From 6b506ce20daa9f38dcc77bc460f5ccccb364bc28 Mon Sep 17 00:00:00 2001 From: Skal Date: Wed, 5 Feb 2020 09:17:18 +0100 Subject: [PATCH] Add -xmp / -exif / -icc options To force-include some binary files as metadata. Also: * switch to C++-11 * add MD5Digest() [will be useful later for Extended-XMP] Change-Id: I517e5fd2ed1054947fdfd079b15878c1f1c986f1 --- CMakeLists.txt | 2 + Makefile | 5 ++ examples/sjpeg.cc | 37 ++++++++++--- examples/utils.cc | 8 +++ examples/utils.h | 2 + man/sjpeg.1 | 15 ++++++ src/md5sum.h | 131 ++++++++++++++++++++++++++++++++++++++++++++++ tests/test_cmd.sh | 56 ++++++++++++++++---- 8 files changed, 239 insertions(+), 17 deletions(-) create mode 100644 src/md5sum.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b00482..992caa3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,6 @@ cmake_minimum_required(VERSION 2.8.7) project(sjpeg CXX) +set(CMAKE_CXX_STANDARD 11) # Options for coder / decoder executables. option(SJPEG_ENABLE_SIMD "Enable any SIMD optimization." ON) @@ -45,6 +46,7 @@ add_library(sjpeg ${CMAKE_CURRENT_SOURCE_DIR}/src/bit_writer.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/enc.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/fdct.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/headers.cc + ${CMAKE_CURRENT_SOURCE_DIR}/src/md5sum.h ${CMAKE_CURRENT_SOURCE_DIR}/src/jpeg_tools.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/score_7.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/sjpeg.h diff --git a/Makefile b/Makefile index cbac8c1..cf04dd1 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,10 @@ ARCHIVE_FILE=sjpeg-$(VERSION).tar.gz EXTRA_FLAGS= -DSJPEG_HAVE_PNG -DSJPEG_HAVE_JPEG UTILS_LIBS= -lpng -ljpeg +# we use C++-11 +EXTRA_FLAGS += -std=c++11 + +# OpenGL and GLUT ifeq ($(strip $(shell uname)), Darwin) EXTRA_FLAGS += -I/opt/local/include EXTRA_FLAGS += -Wno-deprecated-declarations @@ -180,6 +184,7 @@ DIST_FILES= \ src/fdct.cc \ src/headers.cc \ src/jpeg_tools.cc \ + src/md5sum.h \ src/score_7.cc \ src/sjpeg.h \ src/sjpegi.h \ diff --git a/examples/sjpeg.cc b/examples/sjpeg.cc index d800d3b..85a3134 100644 --- a/examples/sjpeg.cc +++ b/examples/sjpeg.cc @@ -53,15 +53,15 @@ static void PrintMatrix(const char name[], const uint8_t m[64], static void PrintMetadataInfo(const EncoderParam& param) { if (!param.iccp.empty()) { - fprintf(stdout, "ICCP: %6u bytes (CRC32: 0x%.8x)\n", + fprintf(stdout, "ICCP: %u bytes \t(CRC32: 0x%.8x)\n", static_cast(param.iccp.size()), GetCRC32(param.iccp)); } if (!param.exif.empty()) { - fprintf(stdout, "EXIF: %6u bytes (CRC32: 0x%.8x)\n", + fprintf(stdout, "EXIF: %u bytes \t(CRC32: 0x%.8x)\n", static_cast(param.exif.size()), GetCRC32(param.exif)); } if (!param.xmp.empty()) { - fprintf(stdout, "XMP: %6u bytes (CRC32: 0x%.8x)\n", + fprintf(stdout, "XMP: %u bytes \t(CRC32: 0x%.8x)\n", static_cast(param.xmp.size()), GetCRC32(param.xmp)); } } @@ -76,6 +76,9 @@ static const char* kNoYes[2] = { "no", "yes" }; int main(int argc, char * argv[]) { const char* input_file = nullptr; const char* output_file = nullptr; + const char* xmp_file = nullptr; + const char* icc_file = nullptr; + const char* exif_file = nullptr; EncoderParam param; float reduction = 100; float quality = 75; @@ -87,6 +90,7 @@ int main(int argc, char * argv[]) { bool quiet = false; bool short_output = false; bool print_crc = false; + bool print_md5 = false; float riskiness = 0; SjpegYUVMode yuv_mode_rec = SJPEG_YUV_AUTO; const char* const usage = @@ -103,10 +107,13 @@ int main(int argc, char * argv[]) { " -psnr ... target YUV-PSNR\n" " -estimate ....... Just estimate and print the JPEG source quality.\n" " -i .............. Just print some information about the input file.\n" + " -xmp ....| Specify the output's metadata with the supplied\n" + " -exif ...| file's content. Warning, this may discard the\n" + " -icc ....| source's content!\n" " -version ........ Print the version and exit.\n" " -quiet .......... Quiet mode. Just save the file.\n" " -short .......... Print shorter 1-line info.\n" - " -crc ............ Just print the output checksum and exit.\n" + " -crc / -md5 ..... Just print the output checksum or MD5 sum and exit.\n" "\n" "Advanced options:\n" " -yuv_mode .......... YUV mode to use:\n" @@ -166,6 +173,12 @@ int main(int argc, char * argv[]) { argv[c - 1], argv[c]); return 1; } + } else if (!strcmp(argv[c], "-xmp") && c + 1 < argc) { + xmp_file = argv[++c]; + } else if (!strcmp(argv[c], "-exif") && c + 1 < argc) { + exif_file = argv[++c]; + } else if (!strcmp(argv[c], "-icc") && c + 1 < argc) { + icc_file = argv[++c]; } else if (!strcmp(argv[c], "-estimate")) { estimate = true; } else if (!strcmp(argv[c], "-no_limit")) { @@ -218,6 +231,8 @@ int main(int argc, char * argv[]) { short_output = true; } else if (!strcmp(argv[c], "-crc")) { print_crc = true; + } else if (!strcmp(argv[c], "-md5")) { + print_md5 = true; } else if (!strcmp(argv[c], "-version")) { const uint32_t version = SjpegVersion(); fprintf(stdout, "%d.%d.%d\n", @@ -240,7 +255,7 @@ int main(int argc, char * argv[]) { param.passes = 10; } // Read input file into the buffer in_bytes[] - std::string input = ReadFile(input_file); + const std::string input = ReadFile(input_file); if (input.size() == 0) return 1; const ImageType input_type = GuessImageType(input); @@ -279,7 +294,11 @@ int main(int argc, char * argv[]) { vector in_bytes = ReadImage(input, &W, &H, ¶m); if (in_bytes.size() == 0) return 1; - if (!short_output && !quiet && !print_crc) { + if (xmp_file != nullptr) param.xmp = ReadFile(xmp_file); + if (icc_file != nullptr) param.iccp = ReadFile(icc_file); + if (exif_file != nullptr) param.exif = ReadFile(exif_file); + + if (!short_output && !quiet && !print_crc && !print_md5) { fprintf(stdout, "Input [%s]: %s (%u bytes, %d x %d)\n", ImageTypeName(input_type), input_file, static_cast(input.size()), @@ -301,7 +320,7 @@ int main(int argc, char * argv[]) { PrintMetadataInfo(param); } } - if (info && !print_crc) return 0; // done + if (info && !print_crc && !print_md5) return 0; // done // finish setting up the quantization matrices if (limit_quantization == false) param.SetLimitQuantization(false); @@ -322,6 +341,10 @@ int main(int argc, char * argv[]) { printf("0x%.8x\n", GetCRC32(out)); return 0; } + if (print_md5) { + printf("%s\n", GetMD5Digest(out).c_str()); + return 0; + } if (!short_output && !quiet) { const bool show_reduction = use_reduction && !use_search; diff --git a/examples/utils.cc b/examples/utils.cc index 441307f..cbf09b2 100644 --- a/examples/utils.cc +++ b/examples/utils.cc @@ -30,6 +30,8 @@ #include // note: this must be included *after* png.h #endif // SJPEG_HAVE_PNG +#include "../src/md5sum.h" + using std::vector; using sjpeg::EncoderParam; @@ -116,6 +118,12 @@ uint32_t GetCRC32(const std::string& data, uint32_t crc) { return ~crc; } +//////////////////////////////////////////////////////////////////////////////// + +std::string GetMD5Digest(const std::string& data) { + return sjpeg::MD5Digest(data).Get(); +} + //////////////////////////////////////////////////////////////////////////////// // JPEG reading diff --git a/examples/utils.h b/examples/utils.h index 7f60fb8..33552bd 100755 --- a/examples/utils.h +++ b/examples/utils.h @@ -40,6 +40,8 @@ extern std::vector ReadImage(const std::string& in, // Return CRC32 signature for data block. 'crc' is the current checksum value. extern uint32_t GetCRC32(const std::string& data, uint32_t crc = 0); +std::string GetMD5Digest(const std::string& data); + /////////////////////////////////////////////////////////////////////////////// // guessed image types diff --git a/man/sjpeg.1 b/man/sjpeg.1 index a158cb5..30dd3a1 100644 --- a/man/sjpeg.1 +++ b/man/sjpeg.1 @@ -101,6 +101,18 @@ If the source is a JPEG file, print the estimated quality and exit. .B \-i, \-info Print some information about the input file and exit. .TP +.BI \-xmp " string +Specify the output XMP metadata. Warning, this will overwrite any data +available from the source. No content validation is performed. +.TP +.BI \-exif " string +Specify the output EXIF metadata. Warning, this will overwrite any data +available from the source. No content validation is performed. +.TP +.BI \-icc " string +Specify the output Color Profile metadata. Warning, this will overwrite +any data available from the source. No content validation is performed. +.TP .B \-quiet Disable information message (error messages are still printed). .TP @@ -110,6 +122,9 @@ Only display short information about the output result. .B \-crc Only display the CRC32 checksum of the output and exit. .TP +.B \-md5 +Only display the MD5 digest of the output and exit. +.TP .B \-no_limit Disable the quantization limit (in case of JPEG input file) allowing to recompress the source at a higher quality factor (result is uncertain, diff --git a/src/md5sum.h b/src/md5sum.h new file mode 100644 index 0000000..0bd2038 --- /dev/null +++ b/src/md5sum.h @@ -0,0 +1,131 @@ +// Copyright 2020 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Simple MD5 Digest code +// +// Author: Skal (pascal.massimino@gmail.com) + +#ifndef SJPEG_MD5SUM_H_ +#define SJPEG_MD5SUM_H_ + +#include +#include +#include + +namespace sjpeg { + +class MD5Digest { + public: + explicit MD5Digest(const std::string& data = "") + : A(0x67452301), B(0xefcdab89), C(0x98badcfe), D(0x10325476) { + uint32_t s = data.size(); + assert(data.size() < (1ull << 32)); + uint32_t i, j; + for (i = 0; i + 64 <= s; i += 64) Add((const uint8_t*)&data[i]); + uint8_t block[64 + 64]; + for (j = 0; i < s; ++i) block[j++] = data[i]; + block[j++] = 0x80; // bit 1 + while ((j & 63) != 56) block[j++] = 0; // pad + for (i = 0, s *= 8; i < 8; ++i, s >>= 8) block[j++] = s & 0xff; + Add(block); + if (j > 64) Add(block + 64); + } + static uint32_t Rotate(uint32_t v, int n) { // n != 0 + return (v << n) | (v >> (32 - n)); + } + static uint32_t Get32(const uint8_t b[64], uint32_t i) { + b += 4 * (i & 15); + return ((uint32_t)b[0] << 0) | ((uint32_t)b[1] << 8) | + ((uint32_t)b[2] << 16) | ((uint32_t)b[3] << 24); + } + static void Put32(uint8_t* b, uint32_t v) { + char tmp[3]; + for (uint32_t i = 0; i < 4; ++i, v >>= 8) { + snprintf(tmp, sizeof(tmp), "%.2X", v & 0xff); + *b++ = tmp[0]; + *b++ = tmp[1]; + } + } + + void Add(const uint8_t block[64]) { + const uint8_t Kr[64] = { + 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, + 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, + 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, + 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21 + }; + const uint32_t KK[64] = { // (1ul << 32) * abs(std::sin(i + 1.)) + 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, + 0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501, + 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be, + 0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821, + 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa, + 0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8, + 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed, + 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a, + 0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c, + 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70, + 0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x04881d05, + 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665, + 0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039, + 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1, + 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1, + 0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391 + }; + uint32_t a = A, b = B, c = C, d = D; + for (uint32_t i = 0; i < 64; ++i) { + uint32_t e = a + KK[i]; + if (i < 16) { + e += d ^ (b & (c ^ d)); + e += Get32(block, i); + } else if (i < 32) { + e += c ^ (d & (b ^ c)); + e += Get32(block, 5 * i + 1); + } else if (i < 48) { + e += b ^ c ^ d; + e += Get32(block, 3 * i + 5); + } else { + e += c ^ (b | ~d); + e += Get32(block, 7 * i); + } + a = d; + d = c; + c = b; + b += Rotate(e, Kr[i]); + } + A += a; + B += b; + C += c; + D += d; + } + + std::string Get() const { // returns the hex digest (upper case) + uint8_t tmp[32]; + Get(tmp); + return std::string((const char*)tmp, 32u); + } + void Get(uint8_t out[32]) const { + Put32(out + 0, A); + Put32(out + 8, B); + Put32(out + 16, C); + Put32(out + 24, D); + } + + private: + uint32_t A, B, C, D; +}; + +} // namespace sjpeg + +#endif // SJPEG_MD5SUM_H_ diff --git a/tests/test_cmd.sh b/tests/test_cmd.sh index 53eadef..8952870 100755 --- a/tests/test_cmd.sh +++ b/tests/test_cmd.sh @@ -7,42 +7,78 @@ TMP_FILE2=/tmp/test BAD_FILE=/tmp/ SRC_FILE1="./testdata/source1.png" SRC_FILE2="./testdata/source2.jpg" -SRC_FILE3="./testdata/source4.ppm" +SRC_FILE3="./testdata/source3.jpg" # large file +SRC_FILE4="./testdata/source4.ppm" # simple coverage of command line arguments. Positive tests. -echo "POSITIVE TESTS" +echo +echo "=== POSITIVE TESTS ===" +echo + set -e ${SJPEG} -version ${SJPEG} -h ${SJPEG} --help ${SJPEG} ${SRC_FILE1} -o ${TMP_FILE1} -yuv_mode 2 -${SJPEG} ${SRC_FILE1} -o ${TMP_FILE1} -yuv_mode 4 -${SJPEG} ${SRC_FILE1} -o ${TMP_FILE1} -q 3 -no_adapt -no_optim -quiet +${SJPEG} ${SRC_FILE1} -o ${TMP_FILE1} -yuv_mode 3 +${SJPEG} ${SRC_FILE1} -o ${TMP_FILE1} -q 3 -no_adapt -no_optim -quiet -sharp -${SJPEG} ${SRC_FILE1} -o ${TMP_FILE1} -r 30 -no_adapt -no_optim -quiet -${SJPEG} ${SRC_FILE2} -o ${TMP_FILE1} -r 30 -no_adapt -no_optim -quiet -${SJPEG} ${SRC_FILE3} -o ${TMP_FILE1} -r 30 -no_adapt -no_optim -quiet +${SJPEG} ${SRC_FILE1} -o ${TMP_FILE1} -r 30 -no_adapt -no_optim -quiet -420 +${SJPEG} ${SRC_FILE2} -o ${TMP_FILE1} -q 24 -psnr 35 -pass 5 \ + -trellis -adapt_bias -quiet +${SJPEG} ${SRC_FILE4} -o ${TMP_FILE1} -size 24000 -pass -1 -tolerance .2 \ + -444 -quiet ${SJPEG} ${SRC_FILE1} -crc +${SJPEG} ${SRC_FILE1} -estimate +${SJPEG} ${SRC_FILE1} -i + +# test CRC is matching +if [ -x "$(command -v md5)" ]; then + for file in ${SRC_FILE1} ${SRC_FILE2} ${SRC_FILE4}; do + ${SJPEG} ${file} -o ${TMP_FILE1} -quiet + ${SJPEG} ${file} -md5 + md5 ${TMP_FILE1} + done +else + echo "'md5' command is not available. Skipping MD5 test." +fi + +# test -xmp / -exif / -icc +echo "This is a test. We need a looooooooooooong line" > ${TMP_FILE1} +${SJPEG} ${SRC_FILE1} -xmp ${TMP_FILE1} -exif ${TMP_FILE1} -icc ${TMP_FILE1} +echo "LARGE ICC" && ${SJPEG} ${SRC_FILE1} -icc ${SRC_FILE3} \ + -quiet -o ${TMP_FILE1} +${SJPEG} ${TMP_FILE1} -o ${TMP_FILE1} -r 76 # negative tests (should fail) -echo "NEGATIVE TESTS" +echo +echo "=== NEGATIVE TESTS ===" +echo + set +e +${SJPEG} ${SJPEG} -no_adapt -no_optim -quiet ${SJPEG} ${SRC_FILE1} -o ${TMP_FILE1} -yuv_mode -1 -quiet +${SJPEG} ${SRC_FILE1} -o ${TMP_FILE1} -yuv_mode 4 -quiet ${SJPEG} ${SRC_FILE1} -o ${TMP_FILE1} -yuv_mode 99 -quiet -no_metadata ${SJPEG} -q 80 -quiet ${SJPEG} ${SRC_FILE1} -risk -quiet ${SJPEG} ${SRC_FILE1} -o ${SJPEG} -o ${TMP_FILE2} -quiet -${SJPEG} ${SRC_FILE1} -o ${TMP_FILE1} -yuv_mode -1 -quiet -${SJPEG} ${SRC_FILE1} -o ${TMP_FILE1} -yuv_mode 99 -quiet +${SJPEG} ${SRC_FILE1} -r 101 +${SJPEG} ${SRC_FILE1} -r -1 ${SJPEG} ${BAD_FILE} -o ${TMP_FILE1} -quiet ${SJPEG} ${SRC_FILE1} -o ${BAD_FILE} -quiet ${SJPEG} ${SRC_FILE2} -o ${SJPEG} -o ${BAD_FILE} -quiet +# test with large EXIF +echo "LARGE EXIF" && ${SJPEG} ${SRC_FILE1} -exif ${SRC_FILE3} -quiet +# test with large file (>64kb). XMP can't handle larger-than-64k data +echo "LARGE XMP" && ${SJPEG} ${SRC_FILE1} -xmp ${SRC_FILE3} -quiet -o ${TMP_FILE1} + # this test does not work for very low quality values (q<4) for q in `seq 4 100`; do ${SJPEG} -q $q ${SRC_FILE1} -o ${TMP_FILE1} -no_adapt -no_optim &> /dev/null