From 6f764ab637b0b4beeaf2e6a0d8905aaff0491c1d Mon Sep 17 00:00:00 2001 From: Mark Peek Date: Wed, 12 Jun 2024 09:49:44 -0700 Subject: [PATCH 01/30] [dev] allow the use of FreeBSD bhyve virtual disks * Closes #2504. --- src/dev.c | 1 + src/rufus.rc | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/dev.c b/src/dev.c index fa405c42d5c..8e3676364c5 100644 --- a/src/dev.c +++ b/src/dev.c @@ -287,6 +287,7 @@ static __inline BOOL IsVHD(const char* buffer) "Arsenal_________Virtual_", "KernSafeVirtual_________", "Msft____Virtual_Disk____", + "BHYVE__________SATA_DISK", "VMware__VMware_Virtual_S" // Enabled through a cheat mode, as this lists primary disks on VMWare instances }; diff --git a/src/rufus.rc b/src/rufus.rc index 3d7d9a91240..7c0de96e632 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2182" +CAPTION "Rufus 4.6.2183" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2182,0 - PRODUCTVERSION 4,6,2182,0 + FILEVERSION 4,6,2183,0 + PRODUCTVERSION 4,6,2183,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2182" + VALUE "FileVersion", "4.6.2183" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2182" + VALUE "ProductVersion", "4.6.2183" END END BLOCK "VarFileInfo" From 7c6dbdc1cef5a300cb798c1007b6f06545f3118f Mon Sep 17 00:00:00 2001 From: wallrik Date: Thu, 27 Jun 2024 21:40:39 +0200 Subject: [PATCH 02/30] [msic] update Windows To Go docs link * Old link is still fine, but we might as well point to the Windows 10 version. * Closes #2511. --- src/rufus.rc | 10 +++++----- src/wue.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/rufus.rc b/src/rufus.rc index 7c0de96e632..3b9f9cfb381 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2183" +CAPTION "Rufus 4.6.2184" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2183,0 - PRODUCTVERSION 4,6,2183,0 + FILEVERSION 4,6,2184,0 + PRODUCTVERSION 4,6,2184,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2183" + VALUE "FileVersion", "4.6.2184" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2183" + VALUE "ProductVersion", "4.6.2184" END END BLOCK "VarFileInfo" diff --git a/src/wue.c b/src/wue.c index 150b2ba86e7..78bca00b42e 100644 --- a/src/wue.c +++ b/src/wue.c @@ -631,7 +631,7 @@ int SetWinToGoIndex(void) /// /// Setup a Windows To Go drive according to the official Microsoft instructions detailed at: -/// https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-8.1-and-8/jj721578(v=ws.11). +/// https://learn.microsoft.com/en-us/previous-versions/windows/it-pro/windows-10/deployment/windows-to-go/deploy-windows-to-go /// Note that as opposed to the technet guide above we use bcdedit rather than 'unattend.xml' /// to disable the recovery environment. /// From 6d29120b38c68b76abb6ed8d8a6ad8e0af21b6af Mon Sep 17 00:00:00 2001 From: Pete Batard Date: Mon, 8 Jul 2024 13:13:21 +0100 Subject: [PATCH 03/30] [wue] add 'Local' to the list of disallowed local account names * Closes #2493. * Also add 'KRBTGT' as it is mentioned at https://learn.microsoft.com/en-us/windows-server/identity/ad-ds/manage/understand-default-user-accounts#default-local-accounts-in-active-directory along with 'Administrator' and 'Guest' and is probably disallowed too. --- src/rufus.rc | 10 +++++----- src/wue.c | 9 +++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/rufus.rc b/src/rufus.rc index 3b9f9cfb381..4b5cd975505 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2184" +CAPTION "Rufus 4.6.2185" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2184,0 - PRODUCTVERSION 4,6,2184,0 + FILEVERSION 4,6,2185,0 + PRODUCTVERSION 4,6,2185,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2184" + VALUE "FileVersion", "4.6.2185" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2184" + VALUE "ProductVersion", "4.6.2185" END END BLOCK "VarFileInfo" diff --git a/src/wue.c b/src/wue.c index 78bca00b42e..b81fd61b7e7 100644 --- a/src/wue.c +++ b/src/wue.c @@ -60,10 +60,11 @@ extern StrArray modified_files; /// The path of a newly created answer file on success or NULL on error. char* CreateUnattendXml(int arch, int flags) { + const static char* xml_arch_names[5] = { "x86", "amd64", "arm", "arm64" }; + const static char* unallowed_account_names[] = { "Administrator", "Guest", "KRBTGT", "Local" }; static char path[MAX_PATH]; FILE* fd; int i, order; - const char* xml_arch_names[5] = { "x86", "amd64", "arm", "arm64" }; unattend_xml_flags = flags; if (arch < ARCH_X86_32 || arch > ARCH_ARM_64 || flags == 0) { uprintf("Note: No Windows User Experience options selected"); @@ -150,10 +151,10 @@ char* CreateUnattendXml(int arch, int flags) fprintf(fd, " \n"); } if (flags & UNATTEND_SET_USER) { - if ((unattend_username[0] == 0) || (stricmp(unattend_username, "Administrator") == 0) || - (stricmp(unattend_username, "Guest") == 0)) { + for (i = 0; (i < ARRAYSIZE(unallowed_account_names)) && (stricmp(unattend_username, unallowed_account_names[i]) != 0); i++); + if (i < ARRAYSIZE(unallowed_account_names)) { uprintf("WARNING: '%s' is not allowed as local account name - Option ignored", unattend_username); - } else { + } else if (unattend_username[0] != 0) { uprintf("• Use '%s' for local account name", unattend_username); // If we create a local account in unattend.xml, then we can get Windows 11 // 22H2 to skip MSA even if the network is connected during installation. From 83b1e7306242884416beac23e0d977725897e217 Mon Sep 17 00:00:00 2001 From: Pete Batard Date: Mon, 8 Jul 2024 13:22:37 +0100 Subject: [PATCH 04/30] [dev] add exception for Samsung uSD Card devices * Closes #2506. --- src/dev.c | 6 +++--- src/hdd_vs_ufd.h | 1 + src/rufus.rc | 10 +++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/dev.c b/src/dev.c index 8e3676364c5..b223b4bb381 100644 --- a/src/dev.c +++ b/src/dev.c @@ -416,9 +416,9 @@ BOOL GetOpticalMedia(IMG_SAVE* img_save) /* For debugging user reports of HDDs vs UFDs */ //#define FORCED_DEVICE #ifdef FORCED_DEVICE -#define FORCED_VID 0x23A9 -#define FORCED_PID 0xEF18 -#define FORCED_NAME "SCSI DISK USB Device" +#define FORCED_VID 0x04E8 +#define FORCED_PID 0x61ED +#define FORCED_NAME "Samsung uSD Card Reader USB Device" #endif void ClearDrives(void) diff --git a/src/hdd_vs_ufd.h b/src/hdd_vs_ufd.h index 484e8dbfa94..e239cb9d225 100644 --- a/src/hdd_vs_ufd.h +++ b/src/hdd_vs_ufd.h @@ -91,6 +91,7 @@ static str_score_t str_adjust[] = { { "Gadget", -10 }, { "Flash", -10 }, { "SD-CARD", -10 }, + { "uSD Card", -10 }, { "HDD", +20 }, { "SATA", +20 }, { "SCSI", +20 }, diff --git a/src/rufus.rc b/src/rufus.rc index 4b5cd975505..186c7a76d15 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2185" +CAPTION "Rufus 4.6.2186" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2185,0 - PRODUCTVERSION 4,6,2185,0 + FILEVERSION 4,6,2186,0 + PRODUCTVERSION 4,6,2186,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2185" + VALUE "FileVersion", "4.6.2186" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2185" + VALUE "ProductVersion", "4.6.2186" END END BLOCK "VarFileInfo" From 10d33c66588bd31070ddc3b45deb1cd83c0aeae0 Mon Sep 17 00:00:00 2001 From: Pete Batard Date: Mon, 8 Jul 2024 13:56:15 +0100 Subject: [PATCH 05/30] [wue] add TimeZone to regional options replication * Closes #2499. --- src/rufus.rc | 10 +++++----- src/wue.c | 6 ++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/rufus.rc b/src/rufus.rc index 186c7a76d15..007cbedbab2 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2186" +CAPTION "Rufus 4.6.2187" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2186,0 - PRODUCTVERSION 4,6,2186,0 + FILEVERSION 4,6,2187,0 + PRODUCTVERSION 4,6,2187,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2186" + VALUE "FileVersion", "4.6.2187" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2186" + VALUE "ProductVersion", "4.6.2187" END END BLOCK "VarFileInfo" diff --git a/src/wue.c b/src/wue.c index b81fd61b7e7..2cb7839640d 100644 --- a/src/wue.c +++ b/src/wue.c @@ -31,6 +31,7 @@ #include "resource.h" #include "registry.h" #include "msapi_utf8.h" +#include "timezoneapi.h" #include "localization.h" /* Memory leaks detection - define _CRTDBG_MAP_ALLOC as preprocessor macro */ @@ -64,6 +65,7 @@ char* CreateUnattendXml(int arch, int flags) const static char* unallowed_account_names[] = { "Administrator", "Guest", "KRBTGT", "Local" }; static char path[MAX_PATH]; FILE* fd; + TIME_ZONE_INFORMATION tz_info; int i, order; unattend_xml_flags = flags; if (arch < ARCH_X86_32 || arch > ARCH_ARM_64 || flags == 0) { @@ -199,6 +201,10 @@ char* CreateUnattendXml(int arch, int flags) ReadRegistryKeyStr(REGKEY_HKCU, "Keyboard Layout\\Preload\\1")); fprintf(fd, " %s\n", ToLocaleName(GetSystemDefaultLCID())); fprintf(fd, " %s\n", ToLocaleName(GetUserDefaultLCID())); + if (GetTimeZoneInformation(&tz_info) == TIME_ZONE_ID_INVALID) + uprintf("WARNING: Could not retrieve current timezone: %s", WindowsErrorString()); + else + fprintf(fd, " %S\n", tz_info.StandardName); fprintf(fd, " %s\n", ToLocaleName(GetUserDefaultUILanguage())); fprintf(fd, " %s\n", // NB: Officially, this is a REG_MULTI_SZ string From 9b3c11122b3bacccac95cea0ff33235d48d48f36 Mon Sep 17 00:00:00 2001 From: Mohmed abdel-fattah Date: Sat, 13 Jul 2024 17:43:40 +0100 Subject: [PATCH 06/30] [misc] reinstate delay-loading of wininet and virtdisk DLLs * Per #2272 and #1877, MinGW has issues when delay loading libraries, but it is possible to apply a workaround to alleviate them, by redefining DECLSPEC_IMPORT before including the corresponding headers. * This is a bit more tricky to accomplish for virtdisk, as MinGW's windows.h header does include virtdisk.h on its own (rather than expect a formal include as MSVC does), so we have to prevent the virtdisk.h inclusion first, by defining a macro, and then apply our workaround. * Per https://sourceforge.net/p/mingw-w64/mailman/mingw-w64-public/thread/ea87573f-65ea-44a2-b4bb-ca96c0a136ab%40akeo.ie/#msg58793876 we are hoping that this should be a temporary workaround and that the root cause of the issue will be fixed in binutils. * Closes #2513. --- .mingw/Makefile.am | 2 +- .mingw/Makefile.in | 2 +- .mingw/virtdisk.def | 8 +++ .mingw/wininet.def | 12 ++++ .vs/rufus.vcxproj | 32 +++++----- src/Makefile.am | 4 +- src/Makefile.in | 4 +- src/net.c | 139 ++++++++++++++------------------------------ src/process.c | 2 +- src/rufus.rc | 10 ++-- src/vhd.c | 55 ++++++------------ src/vhd.h | 6 ++ 12 files changed, 117 insertions(+), 159 deletions(-) create mode 100644 .mingw/virtdisk.def create mode 100644 .mingw/wininet.def diff --git a/.mingw/Makefile.am b/.mingw/Makefile.am index cb131d61d6e..ec826d27cac 100644 --- a/.mingw/Makefile.am +++ b/.mingw/Makefile.am @@ -19,7 +19,7 @@ TARGET := $(word 1,$(subst -, ,$(TUPLE))) DEF_SUFFIX := $(if $(TARGET:x86_64=),.def,.def64) .PHONY: all -all: dwmapi-delaylib.lib version-delaylib.lib wintrust-delaylib.lib +all: dwmapi-delaylib.lib version-delaylib.lib virtdisk-delaylib.lib wininet-delaylib.lib wintrust-delaylib.lib %.def64: %.def $(AM_V_SED) "s/@.*//" $< >$@ diff --git a/.mingw/Makefile.in b/.mingw/Makefile.in index 0dda8fb176d..117eb9c44a5 100644 --- a/.mingw/Makefile.in +++ b/.mingw/Makefile.in @@ -367,7 +367,7 @@ uninstall-am: .PHONY: all -all: dwmapi-delaylib.lib version-delaylib.lib wintrust-delaylib.lib +all: dwmapi-delaylib.lib version-delaylib.lib virtdisk-delaylib.lib wininet-delaylib.lib wintrust-delaylib.lib %.def64: %.def $(AM_V_SED) "s/@.*//" $< >$@ diff --git a/.mingw/virtdisk.def b/.mingw/virtdisk.def new file mode 100644 index 00000000000..7fafde83f5a --- /dev/null +++ b/.mingw/virtdisk.def @@ -0,0 +1,8 @@ +EXPORTS + AttachVirtualDisk@24 + CreateVirtualDisk@36 + DetachVirtualDisk@12 + GetVirtualDiskInformation@16 + GetVirtualDiskPhysicalPath@12 + GetVirtualDiskOperationProgress@12 + OpenVirtualDisk@24 diff --git a/.mingw/wininet.def b/.mingw/wininet.def new file mode 100644 index 00000000000..c1ca40d080b --- /dev/null +++ b/.mingw/wininet.def @@ -0,0 +1,12 @@ +EXPORTS + HttpQueryInfoA@20 + HttpOpenRequestA@32 + HttpSendRequestA@20 + InternetCloseHandle@4 + InternetConnectA@32 + InternetCrackUrlA@16 + InternetGetConnectedState@8 + InternetGetLastResponseInfoA@12 + InternetOpenA@20 + InternetReadFile@16 + InternetSetOptionA@16 diff --git a/.vs/rufus.vcxproj b/.vs/rufus.vcxproj index 7266623b50f..8c9b40f97ee 100644 --- a/.vs/rufus.vcxproj +++ b/.vs/rufus.vcxproj @@ -133,12 +133,12 @@ /utf-8 $(ExternalCompilerOptions) %(AdditionalOptions) - advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;wintrust.lib;%(AdditionalDependencies) + advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;virtdisk.lib;wininet.lib;wintrust.lib;%(AdditionalDependencies) RequireAdministrator true Windows MachineX86 - advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;wintrust.dll;%(DelayLoadDLLs) + advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;virtdisk.dll;wininet.dll;wintrust.dll;%(DelayLoadDLLs) _UNICODE;UNICODE;%(PreprocessorDefinitions) @@ -162,12 +162,12 @@ /utf-8 $(ExternalCompilerOptions) %(AdditionalOptions) - advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;wintrust.lib;ole32.lib;advapi32.lib;gdi32.lib;shell32.lib;comdlg32.lib;%(AdditionalDependencies) + advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;virtdisk.lib;wininet.lib;wintrust.lib;ole32.lib;advapi32.lib;gdi32.lib;shell32.lib;comdlg32.lib;%(AdditionalDependencies) RequireAdministrator true Windows C:\Program Files (x86)\Windows Kits\10\Lib\10.0.15063.0\um\arm - advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;wintrust.dll;ole32.dll;advapi32.dll;gdi32.dll;shell32.dll;comdlg32.dll;%(DelayLoadDLLs) + advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;virtdisk.dll;wininet.dll;wintrust.dll;ole32.dll;advapi32.dll;gdi32.dll;shell32.dll;comdlg32.dll;%(DelayLoadDLLs) _UNICODE;UNICODE;%(PreprocessorDefinitions) @@ -193,12 +193,12 @@ /utf-8 $(ExternalCompilerOptions) %(AdditionalOptions) - advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;wintrust.lib;ole32.lib;advapi32.lib;gdi32.lib;shell32.lib;comdlg32.lib;%(AdditionalDependencies) + advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;virtdisk.lib;wininet.lib;wintrust.lib;ole32.lib;advapi32.lib;gdi32.lib;shell32.lib;comdlg32.lib;%(AdditionalDependencies) RequireAdministrator true Windows C:\Program Files (x86)\Windows Kits\10\Lib\10.0.16299.0\um\arm64 - advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;wintrust.dll;ole32.dll;advapi32.dll;gdi32.dll;shell32.dll;comdlg32.dll;%(DelayLoadDLLs) + advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;virtdisk.dll;wininet.dll;wintrust.dll;ole32.dll;advapi32.dll;gdi32.dll;shell32.dll;comdlg32.dll;%(DelayLoadDLLs) _UNICODE;UNICODE;%(PreprocessorDefinitions) @@ -229,12 +229,12 @@ /utf-8 $(ExternalCompilerOptions) %(AdditionalOptions) - advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;wintrust.lib;%(AdditionalDependencies) + advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;virtdisk.lib;wininet.lib;wintrust.lib;%(AdditionalDependencies) RequireAdministrator true Windows MachineX64 - advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;wintrust.dll;%(DelayLoadDLLs) + advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;virtdisk.dll;wininet.dll;wintrust.dll;%(DelayLoadDLLs) _UNICODE;UNICODE;%(PreprocessorDefinitions) @@ -260,13 +260,13 @@ true - advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;wintrust.lib;%(AdditionalDependencies) + advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;virtdisk.lib;wininet.lib;wintrust.lib;%(AdditionalDependencies) RequireAdministrator false Windows MachineX86 /BREPRO %(AdditionalOptions) - advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;wintrust.dll;%(DelayLoadDLLs) + advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;virtdisk.dll;wininet.dll;wintrust.dll;%(DelayLoadDLLs) _UNICODE;UNICODE;%(PreprocessorDefinitions) @@ -292,13 +292,13 @@ true - advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;wintrust.lib;ole32.lib;advapi32.lib;gdi32.lib;shell32.lib;comdlg32.lib;%(AdditionalDependencies) + advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;virtdisk.lib;wininet.lib;wintrust.lib;ole32.lib;advapi32.lib;gdi32.lib;shell32.lib;comdlg32.lib;%(AdditionalDependencies) RequireAdministrator false Windows C:\Program Files (x86)\Windows Kits\10\Lib\10.0.15063.0\um\arm /BREPRO %(AdditionalOptions) - advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;wintrust.dll;ole32.dll;advapi32.dll;gdi32.dll;shell32.dll;comdlg32.dll;%(DelayLoadDLLs) + advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;virtdisk.dll;wininet.dll;wintrust.dll;ole32.dll;advapi32.dll;gdi32.dll;shell32.dll;comdlg32.dll;%(DelayLoadDLLs) _UNICODE;UNICODE;%(PreprocessorDefinitions) @@ -326,13 +326,13 @@ true - advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;wintrust.lib;ole32.lib;advapi32.lib;gdi32.lib;shell32.lib;comdlg32.lib;%(AdditionalDependencies) + advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;virtdisk.lib;wininet.lib;wintrust.lib;ole32.lib;advapi32.lib;gdi32.lib;shell32.lib;comdlg32.lib;%(AdditionalDependencies) RequireAdministrator false Windows C:\Program Files (x86)\Windows Kits\10\Lib\10.0.16299.0\um\arm64 /BREPRO %(AdditionalOptions) - advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;wintrust.dll;ole32.dll;advapi32.dll;gdi32.dll;shell32.dll;comdlg32.dll;%(DelayLoadDLLs) + advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;virtdisk.dll;wininet.dll;wintrust.dll;ole32.dll;advapi32.dll;gdi32.dll;shell32.dll;comdlg32.dll;%(DelayLoadDLLs) _UNICODE;UNICODE;%(PreprocessorDefinitions) @@ -363,13 +363,13 @@ true - advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;wintrust.lib;%(AdditionalDependencies) + advapi32.lib;comctl32.lib;crypt32.lib;gdi32.lib;ole32.lib;dwmapi.lib;setupapi.lib;shell32.lib;shlwapi.lib;version.lib;virtdisk.lib;wininet.lib;wintrust.lib;%(AdditionalDependencies) RequireAdministrator false Windows MachineX64 /BREPRO %(AdditionalOptions) - advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;wintrust.dll;%(DelayLoadDLLs) + advapi32.dll;comctl32.dll;crypt32.dll;gdi32.dll;ole32.dll;dwmapi.dll;setupapi.dll;shell32.dll;shlwapi.dll;version.dll;virtdisk.dll;wininet.dll;wintrust.dll;%(DelayLoadDLLs) _UNICODE;UNICODE;%(PreprocessorDefinitions) diff --git a/src/Makefile.am b/src/Makefile.am index ef0d0436d10..6d3dc08aa47 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -4,8 +4,8 @@ NONVULNERABLE_LIBS = -lsetupapi -lole32 -lgdi32 -lshlwapi -lcrypt32 -lcomctl32 - # The following libraries are vulnerable (or have an unknown vulnerability status), so we link using our delay-loaded replacement: # Ideally there would also be virtdisk and wininet as delaylib's below, but the MinGW folks haven't quite sorted out delay-loading # for x86_32 so as soon as you try to call APIs from these, the application will crash! -# See https://github.com/pbatard/rufus/issues/1877#issuecomment-1109683039 as well as https://github.com/pbatard/rufus/issues/2272 -VULNERABLE_LIBS = -ldwmapi-delaylib -lversion-delaylib -lwintrust-delaylib +# See https://github.com/pbatard/rufus/issues/2272 +VULNERABLE_LIBS = -ldwmapi-delaylib -lversion-delaylib -lvirtdisk-delaylib -lwininet-delaylib -lwintrust-delaylib noinst_PROGRAMS = rufus diff --git a/src/Makefile.in b/src/Makefile.in index 70d31a54bb1..003ffb8ed19 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -279,8 +279,8 @@ NONVULNERABLE_LIBS = -lsetupapi -lole32 -lgdi32 -lshlwapi -lcrypt32 -lcomctl32 - # The following libraries are vulnerable (or have an unknown vulnerability status), so we link using our delay-loaded replacement: # Ideally there would also be virtdisk and wininet as delaylib's below, but the MinGW folks haven't quite sorted out delay-loading # for x86_32 so as soon as you try to call APIs from these, the application will crash! -# See https://github.com/pbatard/rufus/issues/1877#issuecomment-1109683039 as well as https://github.com/pbatard/rufus/issues/2272 -VULNERABLE_LIBS = -ldwmapi-delaylib -lversion-delaylib -lwintrust-delaylib +# See https://github.com/pbatard/rufus/issues/2272 +VULNERABLE_LIBS = -ldwmapi-delaylib -lversion-delaylib -lvirtdisk-delaylib -lwininet-delaylib -lwintrust-delaylib AM_V_WINDRES_0 = @echo " RC $@";$(WINDRES) AM_V_WINDRES_1 = $(WINDRES) AM_V_WINDRES_ = $(AM_V_WINDRES_$(AM_DEFAULT_VERBOSITY)) diff --git a/src/net.c b/src/net.c index 682fae895f6..8e486ca08d6 100644 --- a/src/net.c +++ b/src/net.c @@ -24,6 +24,12 @@ #endif #include +// Temporary workaround for MinGW32 delay-loading +// See https://github.com/pbatard/rufus/pull/2513 +#if defined(__MINGW32__) +#undef DECLSPEC_IMPORT +#define DECLSPEC_IMPORT __attribute__((visibility("hidden"))) +#endif #include #include #include @@ -113,14 +119,6 @@ static HINTERNET GetInternetSession(const char* user_agent, BOOL bRetry) HINTERNET hSession = NULL; HRESULT hr = S_FALSE; INetworkListManager* pNetworkListManager; - - PF_TYPE_DECL(WINAPI, HINTERNET, InternetOpenA, (LPCSTR, DWORD, LPCSTR, LPCSTR, DWORD)); - PF_TYPE_DECL(WINAPI, BOOL, InternetSetOptionA, (HINTERNET, DWORD, LPVOID, DWORD)); - PF_TYPE_DECL(WINAPI, BOOL, InternetGetConnectedState, (LPDWORD, DWORD)); - PF_INIT_OR_OUT(InternetOpenA, WinInet); - PF_INIT_OR_OUT(InternetSetOptionA, WinInet); - PF_INIT(InternetGetConnectedState, WinInet); - // Create a NetworkListManager Instance to check the network connection IGNORE_RETVAL(CoInitializeEx(NULL, COINIT_APARTMENTTHREADED | COINIT_DISABLE_OLE1DDE)); hr = CoCreateInstance(&CLSID_NetworkListManager, NULL, CLSCTX_ALL, @@ -131,8 +129,8 @@ static HINTERNET GetInternetSession(const char* user_agent, BOOL bRetry) // INetworkListManager may fail with ERROR_SERVICE_DEPENDENCY_FAIL if the DHCP service // is not running, in which case we must fall back to using InternetGetConnectedState(). // See https://github.com/pbatard/rufus/issues/1801. - if ((hr == HRESULT_FROM_WIN32(ERROR_SERVICE_DEPENDENCY_FAIL)) && (pfInternetGetConnectedState != NULL)) { - InternetConnection = pfInternetGetConnectedState(&dwFlags, 0) ? VARIANT_TRUE : VARIANT_FALSE; + if (hr == HRESULT_FROM_WIN32(ERROR_SERVICE_DEPENDENCY_FAIL)) { + InternetConnection = InternetGetConnectedState(&dwFlags, 0) ? VARIANT_TRUE : VARIANT_FALSE; break; } if (hr == S_OK || !bRetry) @@ -147,16 +145,16 @@ static HINTERNET GetInternetSession(const char* user_agent, BOOL bRetry) static_sprintf(default_agent, APPLICATION_NAME "/%d.%d.%d (Windows NT %lu.%lu%s)", rufus_version[0], rufus_version[1], rufus_version[2], WindowsVersion.Major, WindowsVersion.Minor, is_WOW64() ? "; WOW64" : ""); - hSession = pfInternetOpenA((user_agent == NULL) ? default_agent : user_agent, + hSession = InternetOpenA((user_agent == NULL) ? default_agent : user_agent, INTERNET_OPEN_TYPE_PRECONFIG, NULL, NULL, 0); // Set the timeouts - pfInternetSetOptionA(hSession, INTERNET_OPTION_CONNECT_TIMEOUT, (LPVOID)&dwTimeout, sizeof(dwTimeout)); - pfInternetSetOptionA(hSession, INTERNET_OPTION_SEND_TIMEOUT, (LPVOID)&dwTimeout, sizeof(dwTimeout)); - pfInternetSetOptionA(hSession, INTERNET_OPTION_RECEIVE_TIMEOUT, (LPVOID)&dwTimeout, sizeof(dwTimeout)); + InternetSetOptionA(hSession, INTERNET_OPTION_CONNECT_TIMEOUT, (LPVOID)&dwTimeout, sizeof(dwTimeout)); + InternetSetOptionA(hSession, INTERNET_OPTION_SEND_TIMEOUT, (LPVOID)&dwTimeout, sizeof(dwTimeout)); + InternetSetOptionA(hSession, INTERNET_OPTION_RECEIVE_TIMEOUT, (LPVOID)&dwTimeout, sizeof(dwTimeout)); // Enable gzip and deflate decoding schemes - pfInternetSetOptionA(hSession, INTERNET_OPTION_HTTP_DECODING, (LPVOID)&decodingSupport, sizeof(decodingSupport)); + InternetSetOptionA(hSession, INTERNET_OPTION_HTTP_DECODING, (LPVOID)&decodingSupport, sizeof(decodingSupport)); // Enable HTTP/2 protocol support - pfInternetSetOptionA(hSession, INTERNET_OPTION_ENABLE_HTTP_PROTOCOL, (LPVOID)&dwProtocolSupport, sizeof(dwProtocolSupport)); + InternetSetOptionA(hSession, INTERNET_OPTION_ENABLE_HTTP_PROTOCOL, (LPVOID)&dwProtocolSupport, sizeof(dwProtocolSupport)); out: return hSession; @@ -187,23 +185,6 @@ uint64_t DownloadToFileOrBufferEx(const char* url, const char* file, const char* hostname, sizeof(hostname), 0, NULL, 1, urlpath, sizeof(urlpath), NULL, 1}; uint64_t size = 0, total_size = 0; - // Can't link with wininet.lib because of sideloading issues - // And we can't delay-load wininet.dll with MinGW either because the application simply exits on startup... - PF_TYPE_DECL(WINAPI, BOOL, InternetCrackUrlA, (LPCSTR, DWORD, DWORD, LPURL_COMPONENTSA)); - PF_TYPE_DECL(WINAPI, HINTERNET, InternetConnectA, (HINTERNET, LPCSTR, INTERNET_PORT, LPCSTR, LPCSTR, DWORD, DWORD, DWORD_PTR)); - PF_TYPE_DECL(WINAPI, BOOL, InternetReadFile, (HINTERNET, LPVOID, DWORD, LPDWORD)); - PF_TYPE_DECL(WINAPI, BOOL, InternetCloseHandle, (HINTERNET)); - PF_TYPE_DECL(WINAPI, HINTERNET, HttpOpenRequestA, (HINTERNET, LPCSTR, LPCSTR, LPCSTR, LPCSTR, LPCSTR*, DWORD, DWORD_PTR)); - PF_TYPE_DECL(WINAPI, BOOL, HttpSendRequestA, (HINTERNET, LPCSTR, DWORD, LPVOID, DWORD)); - PF_TYPE_DECL(WINAPI, BOOL, HttpQueryInfoA, (HINTERNET, DWORD, LPVOID, LPDWORD, LPDWORD)); - PF_INIT_OR_OUT(InternetCrackUrlA, WinInet); - PF_INIT_OR_OUT(InternetConnectA, WinInet); - PF_INIT_OR_OUT(InternetReadFile, WinInet); - PF_INIT_OR_OUT(InternetCloseHandle, WinInet); - PF_INIT_OR_OUT(HttpOpenRequestA, WinInet); - PF_INIT_OR_OUT(HttpSendRequestA, WinInet); - PF_INIT_OR_OUT(HttpQueryInfoA, WinInet); - ErrorStatus = 0; DownloadStatus = 404; if (hProgressDialog != NULL) @@ -220,7 +201,7 @@ uint64_t DownloadToFileOrBufferEx(const char* url, const char* file, const char* uprintf("Downloading %s", url); } - if ( (!pfInternetCrackUrlA(url, (DWORD)safe_strlen(url), 0, &UrlParts)) + if ( (!InternetCrackUrlA(url, (DWORD)safe_strlen(url), 0, &UrlParts)) || (UrlParts.lpszHostName == NULL) || (UrlParts.lpszUrlPath == NULL)) { uprintf("Unable to decode URL: %s", WindowsErrorString()); goto out; @@ -233,13 +214,13 @@ uint64_t DownloadToFileOrBufferEx(const char* url, const char* file, const char* goto out; } - hConnection = pfInternetConnectA(hSession, UrlParts.lpszHostName, UrlParts.nPort, NULL, NULL, INTERNET_SERVICE_HTTP, 0, (DWORD_PTR)NULL); + hConnection = InternetConnectA(hSession, UrlParts.lpszHostName, UrlParts.nPort, NULL, NULL, INTERNET_SERVICE_HTTP, 0, (DWORD_PTR)NULL); if (hConnection == NULL) { uprintf("Could not connect to server %s:%d: %s", UrlParts.lpszHostName, UrlParts.nPort, WindowsErrorString()); goto out; } - hRequest = pfHttpOpenRequestA(hConnection, "GET", UrlParts.lpszUrlPath, NULL, NULL, accept_types, + hRequest = HttpOpenRequestA(hConnection, "GET", UrlParts.lpszUrlPath, NULL, NULL, accept_types, INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTP | INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTPS | INTERNET_FLAG_NO_COOKIES | INTERNET_FLAG_NO_UI | INTERNET_FLAG_NO_CACHE_WRITE | INTERNET_FLAG_HYPERLINK | ((UrlParts.nScheme==INTERNET_SCHEME_HTTPS)?INTERNET_FLAG_SECURE:0), (DWORD_PTR)NULL); @@ -248,14 +229,14 @@ uint64_t DownloadToFileOrBufferEx(const char* url, const char* file, const char* goto out; } - if (!pfHttpSendRequestA(hRequest, request_headers, -1L, NULL, 0)) { + if (!HttpSendRequestA(hRequest, request_headers, -1L, NULL, 0)) { uprintf("Unable to send request: %s", WindowsErrorString()); goto out; } // Get the file size dwSize = sizeof(DownloadStatus); - pfHttpQueryInfoA(hRequest, HTTP_QUERY_STATUS_CODE|HTTP_QUERY_FLAG_NUMBER, (LPVOID)&DownloadStatus, &dwSize, NULL); + HttpQueryInfoA(hRequest, HTTP_QUERY_STATUS_CODE|HTTP_QUERY_FLAG_NUMBER, (LPVOID)&DownloadStatus, &dwSize, NULL); if (DownloadStatus != 200) { error_code = ERROR_INTERNET_ITEM_NOT_FOUND; SetLastError(RUFUS_ERROR(error_code)); @@ -263,7 +244,7 @@ uint64_t DownloadToFileOrBufferEx(const char* url, const char* file, const char* goto out; } dwSize = sizeof(strsize); - if (!pfHttpQueryInfoA(hRequest, HTTP_QUERY_CONTENT_LENGTH, (LPVOID)strsize, &dwSize, NULL)) { + if (!HttpQueryInfoA(hRequest, HTTP_QUERY_CONTENT_LENGTH, (LPVOID)strsize, &dwSize, NULL)) { uprintf("Unable to retrieve file length: %s", WindowsErrorString()); goto out; } @@ -302,7 +283,7 @@ uint64_t DownloadToFileOrBufferEx(const char* url, const char* file, const char* // User may have cancelled the download if (IS_ERROR(ErrorStatus)) goto out; - if (!pfInternetReadFile(hRequest, buf, sizeof(buf), &dwDownloaded) || (dwDownloaded == 0)) + if (!InternetReadFile(hRequest, buf, sizeof(buf), &dwDownloaded) || (dwDownloaded == 0)) break; if (hProgressDialog != NULL) UpdateProgressWithInfo(OP_NOOP, MSG_241, size, total_size); @@ -347,11 +328,11 @@ uint64_t DownloadToFileOrBufferEx(const char* url, const char* file, const char* safe_free(*buffer); } if (hRequest) - pfInternetCloseHandle(hRequest); + InternetCloseHandle(hRequest); if (hConnection) - pfInternetCloseHandle(hConnection); + InternetCloseHandle(hConnection); if (hSession) - pfInternetCloseHandle(hSession); + InternetCloseHandle(hSession); SetLastError(error_code); return r ? size : 0; @@ -476,23 +457,6 @@ static DWORD WINAPI CheckForUpdatesThread(LPVOID param) SYSTEMTIME ServerTime, LocalTime; FILETIME FileTime; int64_t local_time = 0, reg_time, server_time, update_interval; - - // Can't link with wininet.lib because of sideloading issues - PF_TYPE_DECL(WINAPI, BOOL, InternetCrackUrlA, (LPCSTR, DWORD, DWORD, LPURL_COMPONENTSA)); - PF_TYPE_DECL(WINAPI, HINTERNET, InternetConnectA, (HINTERNET, LPCSTR, INTERNET_PORT, LPCSTR, LPCSTR, DWORD, DWORD, DWORD_PTR)); - PF_TYPE_DECL(WINAPI, BOOL, InternetReadFile, (HINTERNET, LPVOID, DWORD, LPDWORD)); - PF_TYPE_DECL(WINAPI, BOOL, InternetCloseHandle, (HINTERNET)); - PF_TYPE_DECL(WINAPI, HINTERNET, HttpOpenRequestA, (HINTERNET, LPCSTR, LPCSTR, LPCSTR, LPCSTR, LPCSTR*, DWORD, DWORD_PTR)); - PF_TYPE_DECL(WINAPI, BOOL, HttpSendRequestA, (HINTERNET, LPCSTR, DWORD, LPVOID, DWORD)); - PF_TYPE_DECL(WINAPI, BOOL, HttpQueryInfoA, (HINTERNET, DWORD, LPVOID, LPDWORD, LPDWORD)); - PF_INIT_OR_OUT(InternetCrackUrlA, WinInet); - PF_INIT_OR_OUT(InternetConnectA, WinInet); - PF_INIT_OR_OUT(InternetReadFile, WinInet); - PF_INIT_OR_OUT(InternetCloseHandle, WinInet); - PF_INIT_OR_OUT(HttpOpenRequestA, WinInet); - PF_INIT_OR_OUT(HttpSendRequestA, WinInet); - PF_INIT_OR_OUT(HttpQueryInfoA, WinInet); - verbose = ReadSetting32(SETTING_VERBOSE_UPDATES); // Without this the FileDialog will produce error 0x8001010E when compiled for Vista or later IGNORE_RETVAL(CoInitializeEx(NULL, COINIT_APARTMENTTHREADED | COINIT_DISABLE_OLE1DDE)); @@ -530,7 +494,7 @@ static DWORD WINAPI CheckForUpdatesThread(LPVOID param) PrintInfoDebug(3000, MSG_243); status++; // 1 - if (!pfInternetCrackUrlA(server_url, (DWORD)safe_strlen(server_url), 0, &UrlParts)) + if (!InternetCrackUrlA(server_url, (DWORD)safe_strlen(server_url), 0, &UrlParts)) goto out; hostname[sizeof(hostname)-1] = 0; @@ -540,7 +504,7 @@ static DWORD WINAPI CheckForUpdatesThread(LPVOID param) hSession = GetInternetSession(NULL, FALSE); if (hSession == NULL) goto out; - hConnection = pfInternetConnectA(hSession, UrlParts.lpszHostName, UrlParts.nPort, + hConnection = InternetConnectA(hSession, UrlParts.lpszHostName, UrlParts.nPort, NULL, NULL, INTERNET_SERVICE_HTTP, 0, (DWORD_PTR)NULL); if (hConnection == NULL) goto out; @@ -582,11 +546,11 @@ static DWORD WINAPI CheckForUpdatesThread(LPVOID param) UrlParts.dwUrlPathLength = sizeof(urlpath); for (i=0; i 0); } diff --git a/src/process.c b/src/process.c index 31fc6465f64..0060c3975f6 100644 --- a/src/process.c +++ b/src/process.c @@ -895,7 +895,7 @@ static BOOL IsProcessRunning(uint64_t pid) PF_INIT_OR_OUT(NtClose, NtDll); - status = PhOpenProcess(&hProcess, PROCESS_QUERY_LIMITED_INFORMATION, (HANDLE)pid); + status = PhOpenProcess(&hProcess, PROCESS_QUERY_LIMITED_INFORMATION, (HANDLE)(uintptr_t)pid); if (!NT_SUCCESS(status) || (hProcess == NULL)) return FALSE; if (GetExitCodeProcess(hProcess, &dwExitCode)) diff --git a/src/rufus.rc b/src/rufus.rc index 007cbedbab2..38356a1ef86 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2187" +CAPTION "Rufus 4.6.2188" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2187,0 - PRODUCTVERSION 4,6,2187,0 + FILEVERSION 4,6,2188,0 + PRODUCTVERSION 4,6,2188,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2187" + VALUE "FileVersion", "4.6.2188" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2187" + VALUE "ProductVersion", "4.6.2188" END END BLOCK "VarFileInfo" diff --git a/src/vhd.c b/src/vhd.c index 627d94230c0..4ab0236da0d 100644 --- a/src/vhd.c +++ b/src/vhd.c @@ -17,7 +17,15 @@ * along with this program. If not, see . */ +// MinGW includes virdisk.h in windows.h, but we we don't want that +// because we must apply a delay-loading workaround, and that workaround +// has to apply between the winnt.h include and the virdisk.h include. +// So we define _INC_VIRTDISK, to prevent the virdisk.h include in +// windows.h, and then take care of the workaround (and virtdisk.h +// include) in vhd.h. +#define _INC_VIRTDISK #include +#undef _INC_VIRTDISK #include #include #include @@ -68,9 +76,10 @@ static uint8_t wim_flags = 0; static uint32_t progress_report_mask; static uint64_t progress_offset = 0, progress_total = 100; static wchar_t wmount_path[MAX_PATH] = { 0 }, wmount_track[MAX_PATH] = { 0 }; -static char sevenzip_path[MAX_PATH]; -static BOOL count_files; +static char sevenzip_path[MAX_PATH], physical_path[128] = ""; static int progress_op = OP_FILE_COPY, progress_msg = MSG_267; +static BOOL count_files; +static HANDLE mounted_handle = INVALID_HANDLE_VALUE; static BOOL Get7ZipPath(void) { @@ -897,23 +906,6 @@ BOOL WimApplyImage(const char* image, int index, const char* dst) return dw; } -// VirtDisk API Prototypes since we can't use delay-loading because of MinGW -// See https://github.com/pbatard/rufus/issues/2272#issuecomment-1615976013 -PF_TYPE_DECL(WINAPI, DWORD, CreateVirtualDisk, (PVIRTUAL_STORAGE_TYPE, PCWSTR, - VIRTUAL_DISK_ACCESS_MASK, PSECURITY_DESCRIPTOR, CREATE_VIRTUAL_DISK_FLAG, ULONG, - PCREATE_VIRTUAL_DISK_PARAMETERS, LPOVERLAPPED, PHANDLE)); -PF_TYPE_DECL(WINAPI, DWORD, OpenVirtualDisk, (PVIRTUAL_STORAGE_TYPE, PCWSTR, - VIRTUAL_DISK_ACCESS_MASK, OPEN_VIRTUAL_DISK_FLAG, POPEN_VIRTUAL_DISK_PARAMETERS, PHANDLE)); -PF_TYPE_DECL(WINAPI, DWORD, AttachVirtualDisk, (HANDLE, PSECURITY_DESCRIPTOR, - ATTACH_VIRTUAL_DISK_FLAG, ULONG, PATTACH_VIRTUAL_DISK_PARAMETERS, LPOVERLAPPED)); -PF_TYPE_DECL(WINAPI, DWORD, DetachVirtualDisk, (HANDLE, DETACH_VIRTUAL_DISK_FLAG, ULONG)); -PF_TYPE_DECL(WINAPI, DWORD, GetVirtualDiskPhysicalPath, (HANDLE, PULONG, PWSTR)); -PF_TYPE_DECL(WINAPI, DWORD, GetVirtualDiskOperationProgress, (HANDLE, LPOVERLAPPED, PVIRTUAL_DISK_PROGRESS)); -PF_TYPE_DECL(WINAPI, DWORD, GetVirtualDiskInformation, (HANDLE, PULONG, PGET_VIRTUAL_DISK_INFO, PULONG)); - -static char physical_path[128] = ""; -static HANDLE mounted_handle = INVALID_HANDLE_VALUE; - // Mount an ISO or a VHD/VHDX image and provide its size // Returns the physical path of the mounted image or NULL on error. char* VhdMountImageAndGetSize(const char* path, uint64_t* disk_size) @@ -927,12 +919,6 @@ char* VhdMountImageAndGetSize(const char* path, uint64_t* disk_size) wconvert(path); char *ret = NULL, *ext = NULL; - PF_INIT_OR_OUT(OpenVirtualDisk, VirtDisk); - PF_INIT_OR_OUT(AttachVirtualDisk, VirtDisk); - PF_INIT_OR_OUT(GetVirtualDiskPhysicalPath, VirtDisk); - if (disk_size != NULL) - PF_INIT_OR_OUT(GetVirtualDiskInformation, VirtDisk); - if (wpath == NULL) return NULL; @@ -946,7 +932,7 @@ char* VhdMountImageAndGetSize(const char* path, uint64_t* disk_size) else if (safe_stricmp(ext, ".vhd") == 0) vtype.DeviceId = VIRTUAL_STORAGE_TYPE_DEVICE_VHD; - r = pfOpenVirtualDisk(&vtype, wpath, VIRTUAL_DISK_ACCESS_READ | VIRTUAL_DISK_ACCESS_GET_INFO, + r = OpenVirtualDisk(&vtype, wpath, VIRTUAL_DISK_ACCESS_READ | VIRTUAL_DISK_ACCESS_GET_INFO, OPEN_VIRTUAL_DISK_FLAG_NONE, NULL, &mounted_handle); if (r != ERROR_SUCCESS) { SetLastError(r); @@ -955,7 +941,7 @@ char* VhdMountImageAndGetSize(const char* path, uint64_t* disk_size) } vparams.Version = ATTACH_VIRTUAL_DISK_VERSION_1; - r = pfAttachVirtualDisk(mounted_handle, NULL, ATTACH_VIRTUAL_DISK_FLAG_READ_ONLY | + r = AttachVirtualDisk(mounted_handle, NULL, ATTACH_VIRTUAL_DISK_FLAG_READ_ONLY | ATTACH_VIRTUAL_DISK_FLAG_NO_DRIVE_LETTER, 0, &vparams, NULL); if (r != ERROR_SUCCESS) { SetLastError(r); @@ -963,7 +949,7 @@ char* VhdMountImageAndGetSize(const char* path, uint64_t* disk_size) goto out; } - r = pfGetVirtualDiskPhysicalPath(mounted_handle, &size, wtmp); + r = GetVirtualDiskPhysicalPath(mounted_handle, &size, wtmp); if (r != ERROR_SUCCESS) { SetLastError(r); uprintf("Could not obtain physical path for mounted image '%s': %s", path, WindowsErrorString()); @@ -975,7 +961,7 @@ char* VhdMountImageAndGetSize(const char* path, uint64_t* disk_size) *disk_size = 0; disk_info.Version = GET_VIRTUAL_DISK_INFO_SIZE; size = sizeof(disk_info); - r = pfGetVirtualDiskInformation(mounted_handle, &size, &disk_info, NULL); + r = GetVirtualDiskInformation(mounted_handle, &size, &disk_info, NULL); if (r != ERROR_SUCCESS) { SetLastError(r); uprintf("Could not obtain virtual size of mounted image '%s': %s", path, WindowsErrorString()); @@ -995,12 +981,10 @@ char* VhdMountImageAndGetSize(const char* path, uint64_t* disk_size) void VhdUnmountImage(void) { - PF_INIT_OR_OUT(DetachVirtualDisk, VirtDisk); - if ((mounted_handle == NULL) || (mounted_handle == INVALID_HANDLE_VALUE)) goto out; - pfDetachVirtualDisk(mounted_handle, DETACH_VIRTUAL_DISK_FLAG_NONE, 0); + DetachVirtualDisk(mounted_handle, DETACH_VIRTUAL_DISK_FLAG_NONE, 0); safe_closehandle(mounted_handle); out: physical_path[0] = 0; @@ -1022,9 +1006,6 @@ static DWORD WINAPI VhdSaveImageThread(void* param) OVERLAPPED overlapped = { 0 }; DWORD r = ERROR_NOT_FOUND, flags; - PF_INIT_OR_OUT(CreateVirtualDisk, VirtDisk); - PF_INIT_OR_OUT(GetVirtualDiskOperationProgress, VirtDisk); - assert(img_save->Type == VIRTUAL_STORAGE_TYPE_DEVICE_VHD || img_save->Type == VIRTUAL_STORAGE_TYPE_DEVICE_VHDX); @@ -1052,7 +1033,7 @@ static DWORD WINAPI VhdSaveImageThread(void* param) // CreateVirtualDisk() does not have an overwrite flag... DeleteFileW(wDst); - r = pfCreateVirtualDisk(&vtype, wDst, VIRTUAL_DISK_ACCESS_NONE, NULL, + r = CreateVirtualDisk(&vtype, wDst, VIRTUAL_DISK_ACCESS_NONE, NULL, flags, 0, (PCREATE_VIRTUAL_DISK_PARAMETERS)&vparams, &overlapped, &handle); if (r != ERROR_SUCCESS && r != ERROR_IO_PENDING) { SetLastError(r); @@ -1066,7 +1047,7 @@ static DWORD WINAPI VhdSaveImageThread(void* param) CancelIoEx(handle, &overlapped); goto out; } - if (pfGetVirtualDiskOperationProgress(handle, &overlapped, &vprogress) == ERROR_SUCCESS) { + if (GetVirtualDiskOperationProgress(handle, &overlapped, &vprogress) == ERROR_SUCCESS) { if (vprogress.OperationStatus == ERROR_IO_PENDING) UpdateProgressWithInfo(OP_FORMAT, MSG_261, vprogress.CurrentValue, vprogress.CompletionValue); } diff --git a/src/vhd.h b/src/vhd.h index 45a332b227a..75fa1c03291 100644 --- a/src/vhd.h +++ b/src/vhd.h @@ -19,6 +19,12 @@ #include #include +// Temporary workaround for MinGW32 delay-loading +// See https://github.com/pbatard/rufus/pull/2513 +#if defined(__MINGW32__) +#undef DECLSPEC_IMPORT +#define DECLSPEC_IMPORT __attribute__((visibility("hidden"))) +#endif #include #pragma once From 78608c35fedac8f4e59497e53c1326799b3b2a58 Mon Sep 17 00:00:00 2001 From: Pete Batard Date: Mon, 15 Jul 2024 17:35:08 +0100 Subject: [PATCH 07/30] [wue] fix TimeZone option not being applied to the right parent * Closes #2519. * Also update MinGW's base WINVER to Windows 10. --- configure | 2 +- configure.ac | 2 +- src/rufus.h | 2 +- src/rufus.rc | 10 +++++----- src/vhd.h | 4 ---- src/wue.c | 14 ++++++++++---- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/configure b/configure index 0c1f38d5c7a..667af6b2f0b 100755 --- a/configure +++ b/configure @@ -4638,7 +4638,7 @@ esac printf "%s\n" "#define _GNU_SOURCE /**/" >>confdefs.h -AM_CFLAGS="${AM_CFLAGS} -DWINVER=0x601 -D_WIN32_WINNT=0x601 -D_WIN32_IE=0x800" +AM_CFLAGS="${AM_CFLAGS} -DWINVER=0x0A00 -D_WIN32_WINNT=0x0A00 -D_WIN32_IE=0x0A00" # "-Wl,--nxcompat" to enable DEP (Data Execution Prevention) # "-Wl,--dynamicbase" to enable ASLR (Address Space Layout Randomization) AM_LDFLAGS="${AM_LDFLAGS} -Wl,-no-undefined -Wl,--nxcompat -Wl,--no-insert-timestamp -Wl,--dynamicbase" diff --git a/configure.ac b/configure.ac index 2d757d0aa15..513ab26d6cf 100644 --- a/configure.ac +++ b/configure.ac @@ -18,7 +18,7 @@ AC_CHECK_TOOL(WINDRES, windres, :) AC_C_INLINE AC_DEFINE([_GNU_SOURCE], [], [Use GNU extensions]) -AM_CFLAGS="${AM_CFLAGS} -DWINVER=0x601 -D_WIN32_WINNT=0x601 -D_WIN32_IE=0x800" +AM_CFLAGS="${AM_CFLAGS} -DWINVER=0x0A00 -D_WIN32_WINNT=0x0A00 -D_WIN32_IE=0x0A00" # "-Wl,--nxcompat" to enable DEP (Data Execution Prevention) # "-Wl,--dynamicbase" to enable ASLR (Address Space Layout Randomization) AM_LDFLAGS="${AM_LDFLAGS} -Wl,-no-undefined -Wl,--nxcompat -Wl,--no-insert-timestamp -Wl,--dynamicbase" diff --git a/src/rufus.h b/src/rufus.h index 86e09a7797e..5007f9bd7d3 100644 --- a/src/rufus.h +++ b/src/rufus.h @@ -627,7 +627,7 @@ typedef struct { #define UNATTEND_WINPE_SETUP_MASK (UNATTEND_SECUREBOOT_TPM_MINRAM) #define UNATTEND_SPECIALIZE_DEPLOYMENT_MASK (UNATTEND_NO_ONLINE_ACCOUNT) -#define UNATTEND_OOBE_SHELL_SETUP_MASK (UNATTEND_NO_DATA_COLLECTION | UNATTEND_SET_USER) +#define UNATTEND_OOBE_SHELL_SETUP_MASK (UNATTEND_NO_DATA_COLLECTION | UNATTEND_SET_USER | UNATTEND_DUPLICATE_LOCALE) #define UNATTEND_OOBE_INTERNATIONAL_MASK (UNATTEND_DUPLICATE_LOCALE) #define UNATTEND_OOBE_MASK (UNATTEND_OOBE_SHELL_SETUP_MASK | UNATTEND_OOBE_INTERNATIONAL_MASK | UNATTEND_DISABLE_BITLOCKER) #define UNATTEND_OFFLINE_SERVICING_MASK (UNATTEND_OFFLINE_INTERNAL_DRIVES | UNATTEND_FORCE_S_MODE) diff --git a/src/rufus.rc b/src/rufus.rc index 38356a1ef86..8392db42f09 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2188" +CAPTION "Rufus 4.6.2189" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2188,0 - PRODUCTVERSION 4,6,2188,0 + FILEVERSION 4,6,2189,0 + PRODUCTVERSION 4,6,2189,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2188" + VALUE "FileVersion", "4.6.2189" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2188" + VALUE "ProductVersion", "4.6.2189" END END BLOCK "VarFileInfo" diff --git a/src/vhd.h b/src/vhd.h index 75fa1c03291..07e2c74e9d9 100644 --- a/src/vhd.h +++ b/src/vhd.h @@ -60,10 +60,6 @@ #define MBR_SIZE 512 // Might need to review this once we see bootable 4k systems -// TODO: Remove this once MinGW has been updated -#ifndef VIRTUAL_STORAGE_TYPE_DEVICE_VHDX -#define VIRTUAL_STORAGE_TYPE_DEVICE_VHDX 3 -#endif #define VIRTUAL_STORAGE_TYPE_DEVICE_FFU 99 #define CREATE_VIRTUAL_DISK_VERSION_2 2 #define CREATE_VIRTUAL_DISK_FLAG_CREATE_BACKING_STORAGE 8 diff --git a/src/wue.c b/src/wue.c index 2cb7839640d..b257cb67cd9 100644 --- a/src/wue.c +++ b/src/wue.c @@ -64,6 +64,7 @@ char* CreateUnattendXml(int arch, int flags) const static char* xml_arch_names[5] = { "x86", "amd64", "arm", "arm64" }; const static char* unallowed_account_names[] = { "Administrator", "Guest", "KRBTGT", "Local" }; static char path[MAX_PATH]; + char* tzstr; FILE* fd; TIME_ZONE_INFORMATION tz_info; int i, order; @@ -152,6 +153,15 @@ char* CreateUnattendXml(int arch, int flags) fprintf(fd, " 3\n"); fprintf(fd, " \n"); } + if (flags & UNATTEND_DUPLICATE_LOCALE) { + if ((GetTimeZoneInformation(&tz_info) == TIME_ZONE_ID_INVALID) || + ((tzstr = wchar_to_utf8(tz_info.StandardName)) == NULL)) { + uprintf("WARNING: Could not retrieve current timezone: %s", WindowsErrorString()); + } else { + fprintf(fd, " %s\n", tzstr); + free(tzstr); + } + } if (flags & UNATTEND_SET_USER) { for (i = 0; (i < ARRAYSIZE(unallowed_account_names)) && (stricmp(unattend_username, unallowed_account_names[i]) != 0); i++); if (i < ARRAYSIZE(unallowed_account_names)) { @@ -201,10 +211,6 @@ char* CreateUnattendXml(int arch, int flags) ReadRegistryKeyStr(REGKEY_HKCU, "Keyboard Layout\\Preload\\1")); fprintf(fd, " %s\n", ToLocaleName(GetSystemDefaultLCID())); fprintf(fd, " %s\n", ToLocaleName(GetUserDefaultLCID())); - if (GetTimeZoneInformation(&tz_info) == TIME_ZONE_ID_INVALID) - uprintf("WARNING: Could not retrieve current timezone: %s", WindowsErrorString()); - else - fprintf(fd, " %S\n", tz_info.StandardName); fprintf(fd, " %s\n", ToLocaleName(GetUserDefaultUILanguage())); fprintf(fd, " %s\n", // NB: Officially, this is a REG_MULTI_SZ string From d5302c0bad94f808f0b192599ddc6b93436c5e91 Mon Sep 17 00:00:00 2001 From: Pete Batard Date: Wed, 17 Jul 2024 14:30:59 +0100 Subject: [PATCH 08/30] [misc] fix Coverity warnings * Also use a new if_not_assert() construct where possible. --- src/badblocks.c | 11 +++++++- src/bled/crc32.c | 2 ++ src/bled/decompress_gunzip.c | 14 ++++++++-- src/bled/libbb.h | 1 + src/bled/xz_config.h | 1 + src/bled/xz_dec_bcj.c | 1 + src/bled/xz_dec_lzma2.c | 7 +++-- src/dev.c | 18 ++++++++----- src/drive.c | 12 ++++++--- src/ext2fs/dirhash.c | 1 + src/ext2fs/initialize.c | 5 +++- src/format.c | 50 +++++++++++++++++++++++++----------- src/iso.c | 9 ++++--- src/process.c | 9 +++---- src/registry.h | 15 ++++------- src/rufus.c | 6 ++--- src/rufus.h | 3 +++ src/rufus.rc | 10 ++++---- src/stdfn.c | 12 ++++----- src/stdio.c | 3 ++- src/stdlg.c | 3 ++- src/ui.c | 3 ++- src/vhd.c | 5 ++-- src/wue.c | 3 ++- 24 files changed, 134 insertions(+), 70 deletions(-) diff --git a/src/badblocks.c b/src/badblocks.c index 9fd29d280e8..7a2a3d18b9c 100644 --- a/src/badblocks.c +++ b/src/badblocks.c @@ -426,6 +426,11 @@ static unsigned int test_rw(HANDLE hDrive, blk64_t last_block, size_t block_size cancel_ops = -1; return 0; } + if ((first_block * block_size > 1 * PB) || (last_block * block_size > 1 * PB)) { + uprintf("%sDisk is too large\n", bb_prefix); + cancel_ops = -1; + return 0; + } buffer = allocate_buffer(2 * blocks_at_once * block_size); if (!buffer) { @@ -543,8 +548,12 @@ static unsigned int test_rw(HANDLE hDrive, blk64_t last_block, size_t block_size for (i=0; i < got; i++) { if (memcmp(read_buffer + i * block_size, buffer + i * block_size, - block_size)) + block_size)) { + if_not_assert(currently_testing * block_size < 1 * PB) + goto out; + // coverity[overflow_const] bb_count += bb_output(currently_testing+i-got, CORRUPTION_ERROR); + } } if (v_flag > 1) print_status(); diff --git a/src/bled/crc32.c b/src/bled/crc32.c index ead99050d8b..98e416adc2c 100644 --- a/src/bled/crc32.c +++ b/src/bled/crc32.c @@ -38,6 +38,7 @@ static void crc32init_le(uint32_t *crc32table_le) crc32table_le[0] = 0; for (i = 1 << (CRC_LE_BITS - 1); i; i >>= 1) { + // coverity[overflow_const] crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY_LE : 0); for (j = 0; j < 1 << CRC_LE_BITS; j += 2 * i) crc32table_le[i + j] = crc ^ crc32table_le[j]; @@ -81,6 +82,7 @@ static void crc32init_be(uint32_t *crc32table_be) uint32_t crc = 0x80000000; for (i = 1; i < 1 << CRC_BE_BITS; i <<= 1) { + // coverity[overflow_const] crc = (crc << 1) ^ ((crc & 0x80000000) ? CRCPOLY_BE : 0); for (j = 0; j < i; j++) crc32table_be[i + j] = crc ^ crc32table_be[j]; diff --git a/src/bled/decompress_gunzip.c b/src/bled/decompress_gunzip.c index 1d3d832ed52..6f0ddcf92f0 100644 --- a/src/bled/decompress_gunzip.c +++ b/src/bled/decompress_gunzip.c @@ -280,6 +280,7 @@ static unsigned fill_bitbuffer(STATE_PARAM unsigned bitbuffer, unsigned *current bytebuffer_offset++; *current += 8; } + // coverity[return_overflow] return bitbuffer; } @@ -661,7 +662,7 @@ static NOINLINE int inflate_codes(STATE_PARAM_ONLY) /* called once from inflate_block */ -static void inflate_stored_setup(STATE_PARAM int my_n, int my_b, int my_k) +static void inflate_stored_setup(STATE_PARAM unsigned my_n, unsigned my_b, unsigned my_k) { inflate_stored_n = my_n; inflate_stored_b = my_b; @@ -1043,7 +1044,15 @@ inflate_unzip_internal(STATE_PARAM transformer_state_t *xstate) n = (nwrote <0)?nwrote:-1; goto ret; } - IF_DESKTOP(n += nwrote;) +#if ENABLE_DESKTOP + long long int v = n + nwrote; + if (v < n) { + bb_simple_error_msg("overflow"); + n = -1; + goto ret; + } + n = v; +#endif if (r == 0) break; } @@ -1054,6 +1063,7 @@ inflate_unzip_internal(STATE_PARAM transformer_state_t *xstate) bytebuffer_offset--; bytebuffer[bytebuffer_offset] = gunzip_bb & 0xff; gunzip_bb >>= 8; + // coverity[overflow_const] gunzip_bk -= 8; } ret: diff --git a/src/bled/libbb.h b/src/bled/libbb.h index 7d17823663e..000a1a66d9b 100644 --- a/src/bled/libbb.h +++ b/src/bled/libbb.h @@ -23,6 +23,7 @@ #include "platform.h" #include "msapi_utf8.h" +#include #include #include #include diff --git a/src/bled/xz_config.h b/src/bled/xz_config.h index 05d90af95b3..682ec553c72 100644 --- a/src/bled/xz_config.h +++ b/src/bled/xz_config.h @@ -18,6 +18,7 @@ /* #define XZ_DEC_ARMTHUMB */ /* #define XZ_DEC_SPARC */ +#include #include #include #include diff --git a/src/bled/xz_dec_bcj.c b/src/bled/xz_dec_bcj.c index 219a8bc8799..495d8a3f3c9 100644 --- a/src/bled/xz_dec_bcj.c +++ b/src/bled/xz_dec_bcj.c @@ -109,6 +109,7 @@ static noinline_for_stack size_t XZ_FUNC bcj_x86( if ((buf[i] & 0xFE) != 0xE8) continue; + // coverity[overflow_const] prev_pos = i - prev_pos; if (prev_pos > 3) { prev_mask = 0; diff --git a/src/bled/xz_dec_lzma2.c b/src/bled/xz_dec_lzma2.c index 7d936a83141..ed519d6d8ed 100644 --- a/src/bled/xz_dec_lzma2.c +++ b/src/bled/xz_dec_lzma2.c @@ -622,7 +622,7 @@ static void XZ_FUNC lzma_len(struct xz_dec_lzma2 *s, struct lzma_len_dec *l, uint32_t pos_state) { uint16_t *probs; - uint32_t limit; + uint32_t limit, v; if (!rc_bit(&s->rc, &l->choice)) { probs = l->low[pos_state]; @@ -641,7 +641,9 @@ static void XZ_FUNC lzma_len(struct xz_dec_lzma2 *s, struct lzma_len_dec *l, } } - s->lzma.len += rc_bittree(&s->rc, probs, limit) - limit; + v = s->lzma.len + rc_bittree(&s->rc, probs, limit) - limit; + assert(v >= s->lzma.len); + s->lzma.len = (v < s->lzma.len) ? 0 : v; } /* Decode a match. The distance will be stored in s->lzma.rep0. */ @@ -660,6 +662,7 @@ static void XZ_FUNC lzma_match(struct xz_dec_lzma2 *s, uint32_t pos_state) lzma_len(s, &s->lzma.match_len_dec, pos_state); probs = s->lzma.dist_slot[lzma_get_dist_state(s->lzma.len)]; + // coverity[overflow_const] dist_slot = rc_bittree(&s->rc, probs, DIST_SLOTS) - DIST_SLOTS; if (dist_slot < DIST_MODEL_START) { diff --git a/src/dev.c b/src/dev.c index b223b4bb381..801fb12430e 100644 --- a/src/dev.c +++ b/src/dev.c @@ -137,7 +137,8 @@ BOOL CyclePort(int index) DWORD size; USB_CYCLE_PORT_PARAMS cycle_port; - assert(index < MAX_DRIVES); + if_not_assert(index < MAX_DRIVES) + return -1; // Wait at least 10 secs between resets if (GetTickCount64() < LastReset + 10000ULL) { uprintf("You must wait at least 10 seconds before trying to reset a device"); @@ -190,7 +191,8 @@ int CycleDevice(int index) SP_DEVINFO_DATA dev_info_data; SP_PROPCHANGE_PARAMS propchange_params; - assert(index < MAX_DRIVES); + if_not_assert(index < MAX_DRIVES) + return ERROR_INVALID_DRIVE; if ((index < 0) || (safe_strlen(rufus_drive[index].id) < 8)) return ERROR_INVALID_PARAMETER; @@ -583,8 +585,10 @@ BOOL GetDevices(DWORD devnum) // Better safe than sorry. And yeah, we could have used arrays of // arrays to avoid this, but it's more readable this way. - assert((uasp_start > 0) && (uasp_start < ARRAYSIZE(usbstor_name))); - assert((card_start > 0) && (card_start < ARRAYSIZE(genstor_name))); + if_not_assert((uasp_start > 0) && (uasp_start < ARRAYSIZE(usbstor_name))) + goto out; + if_not_assert((card_start > 0) && (card_start < ARRAYSIZE(genstor_name))) + goto out; devid_list = NULL; if (full_list_size != 0) { @@ -671,7 +675,8 @@ BOOL GetDevices(DWORD devnum) } // Also test for "_SD&" instead of "_SD_" and so on to allow for devices like // "SCSI\DiskRicoh_Storage_SD&REV_3.0" to be detected. - assert(strlen(scsi_card_name_copy) > 1); + if_not_assert(strlen(scsi_card_name_copy) > 1) + continue; scsi_card_name_copy[strlen(scsi_card_name_copy) - 1] = '&'; if (safe_strstr(buffer, scsi_card_name_copy) != NULL) { props.is_CARD = TRUE; @@ -999,7 +1004,8 @@ BOOL GetDevices(DWORD devnum) rufus_drive[num_drives].display_name = safe_strdup(display_name); rufus_drive[num_drives].label = safe_strdup(label); rufus_drive[num_drives].size = drive_size; - assert(rufus_drive[num_drives].size != 0); + if_not_assert(rufus_drive[num_drives].size != 0) + break; if (hub_path != NULL) { rufus_drive[num_drives].hub = safe_strdup(hub_path); rufus_drive[num_drives].port = props.port; diff --git a/src/drive.c b/src/drive.c index e3a9d7cb25c..8d0514d8fde 100644 --- a/src/drive.c +++ b/src/drive.c @@ -280,9 +280,12 @@ char* GetLogicalName(DWORD DriveIndex, uint64_t PartitionOffset, BOOL bKeepTrail // Sanity checks len = safe_strlen(volume_name); - assert(len > 4); - assert(safe_strnicmp(volume_name, volume_start, 4) == 0); - assert(volume_name[len - 1] == '\\'); + if_not_assert(len > 4) + continue; + if_not_assert(safe_strnicmp(volume_name, volume_start, 4) == 0) + continue; + if_not_assert(volume_name[len - 1] == '\\') + continue; drive_type = GetDriveTypeA(volume_name); if ((drive_type != DRIVE_REMOVABLE) && (drive_type != DRIVE_FIXED)) @@ -1817,7 +1820,8 @@ const char* GetFsName(HANDLE hPhysical, LARGE_INTEGER StartingOffset) } } assert(rev < ARRAYSIZE(ext_names)); - ret = ext_names[rev]; + if (rev < ARRAYSIZE(ext_names)) + ret = ext_names[rev]; goto out; } diff --git a/src/ext2fs/dirhash.c b/src/ext2fs/dirhash.c index 42fe98bb3b7..434910ceb2a 100644 --- a/src/ext2fs/dirhash.c +++ b/src/ext2fs/dirhash.c @@ -45,6 +45,7 @@ static void TEA_transform(__u32 buf[4], __u32 const in[]) int n = 16; do { + // coverity[overflow_const] sum += DELTA; b0 += ((b1 << 4)+a) ^ (b1+sum) ^ ((b1 >> 5)+b); b1 += ((b0 << 4)+c) ^ (b0+sum) ^ ((b0 >> 5)+d); diff --git a/src/ext2fs/initialize.c b/src/ext2fs/initialize.c index afe7d6e9a37..04c032d2594 100644 --- a/src/ext2fs/initialize.c +++ b/src/ext2fs/initialize.c @@ -11,6 +11,7 @@ */ #include "config.h" +#include #include #include #if HAVE_UNISTD_H @@ -374,7 +375,9 @@ errcode_t ext2fs_initialize(const char *name, int flags, * adjust inode count to reflect the adjusted inodes_per_group */ if ((__u64)super->s_inodes_per_group * fs->group_desc_count > ~0U) { - ipg--; + assert(ipg != 0); + if (ipg != 0) + ipg--; goto ipg_retry; } super->s_inodes_count = super->s_inodes_per_group * diff --git a/src/format.c b/src/format.c index e468883f069..fe5b2b69508 100644 --- a/src/format.c +++ b/src/format.c @@ -117,8 +117,7 @@ static BOOLEAN __stdcall FormatExCallback(FILE_SYSTEM_CALLBACK_COMMAND Command, if (IS_ERROR(ErrorStatus)) return FALSE; - assert((actual_fs_type >= 0) && (actual_fs_type < FS_MAX)); - if ((actual_fs_type < 0) || (actual_fs_type >= FS_MAX)) + if_not_assert((actual_fs_type >= 0) && (actual_fs_type < FS_MAX)) return FALSE; switch(Command) { @@ -1108,6 +1107,10 @@ static int sector_write(int fd, const void* _buf, unsigned int count) if (sec_size == 0) sec_size = 512; + if_not_assert(sec_size <= 64 * KB) + return -1; + if_not_assert(count <= 1 * GB) + return -1; // If we are on a sector boundary and count is multiple of the // sector size, just issue a regular write @@ -1116,6 +1119,8 @@ static int sector_write(int fd, const void* _buf, unsigned int count) // If we have an existing partial sector, fill and write it if (sec_buf_pos > 0) { + if_not_assert(sec_size >= sec_buf_pos) + return -1; fill_size = min(sec_size - sec_buf_pos, count); memcpy(&sec_buf[sec_buf_pos], buf, fill_size); sec_buf_pos += fill_size; @@ -1133,10 +1138,18 @@ static int sector_write(int fd, const void* _buf, unsigned int count) written = _write(fd, &buf[fill_size], sec_num * sec_size); if (written < 0) return written; - else if (written != sec_num * sec_size) - return fill_size + written; + if (written != sec_num * sec_size) { + // Detect overflows + // coverity[overflow] + int v = fill_size + written; + if_not_assert(v >= fill_size) + return -1; + else + return v; + } sec_buf_pos = count - fill_size - written; - assert(sec_buf_pos < sec_size); + if_not_assert(sec_buf_pos < sec_size) + return -1; // Keep leftover bytes, if any, in the sector buffer if (sec_buf_pos != 0) @@ -1180,7 +1193,8 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive) uprintf("Could not allocate disk zeroing buffer"); goto out; } - assert((uintptr_t)buffer % SelectedDrive.SectorSize == 0); + if_not_assert((uintptr_t)buffer % SelectedDrive.SectorSize == 0) + goto out; // Clear buffer memset(buffer, fast_zeroing ? 0xff : 0x00, buf_size); @@ -1192,7 +1206,8 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive) uprintf("Could not allocate disk comparison buffer"); goto out; } - assert((uintptr_t)cmp_buffer % SelectedDrive.SectorSize == 0); + if_not_assert((uintptr_t)cmp_buffer % SelectedDrive.SectorSize == 0) + goto out; } read_size[0] = buf_size; @@ -1290,7 +1305,8 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive) uprintf("Could not allocate disk write buffer"); goto out; } - assert((uintptr_t)sec_buf % SelectedDrive.SectorSize == 0); + if_not_assert((uintptr_t)sec_buf% SelectedDrive.SectorSize == 0) + goto out; sec_buf_pos = 0; bled_init(256 * KB, uprintf, NULL, sector_write, update_progress, NULL, &ErrorStatus); bled_ret = bled_uncompress_with_handles(hSourceImage, hPhysicalDrive, img_report.compression_type); @@ -1312,7 +1328,8 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive) goto out; } } else { - assert(img_report.compression_type != IMG_COMPRESSION_FFU); + if_not_assert(img_report.compression_type != IMG_COMPRESSION_FFU) + goto out; // VHD/VHDX require mounting the image first if (img_report.compression_type == IMG_COMPRESSION_VHD || img_report.compression_type == IMG_COMPRESSION_VHDX) { @@ -1338,7 +1355,8 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive) uprintf("Could not allocate disk write buffer"); goto out; } - assert((uintptr_t)buffer % SelectedDrive.SectorSize == 0); + if_not_assert((uintptr_t)buffer% SelectedDrive.SectorSize == 0) + goto out; // Start the initial read ReadFileAsync(hSourceImage, &buffer[read_bufnum * buf_size], (DWORD)MIN(buf_size, target_size)); @@ -1365,7 +1383,8 @@ static BOOL WriteDrive(HANDLE hPhysicalDrive, BOOL bZeroDrive) // 2. WriteFile fails unless the size is a multiple of sector size if (read_size[read_bufnum] % SelectedDrive.SectorSize != 0) { - assert(HI_ALIGN_X_TO_Y(read_size[read_bufnum], SelectedDrive.SectorSize) <= buf_size); + if_not_assert(HI_ALIGN_X_TO_Y(read_size[read_bufnum], SelectedDrive.SectorSize) <= buf_size) + goto out; read_size[read_bufnum] = HI_ALIGN_X_TO_Y(read_size[read_bufnum], SelectedDrive.SectorSize); } @@ -1660,7 +1679,8 @@ DWORD WINAPI FormatThread(void* param) if (img_report.compression_type == IMG_COMPRESSION_FFU) { char cmd[MAX_PATH + 128], *physical = NULL; // Should have been filtered out beforehand - assert(has_ffu_support); + if_not_assert(has_ffu_support) + goto out; safe_unlockclose(hPhysicalDrive); physical = GetPhysicalName(SelectedDrive.DeviceNumber); static_sprintf(cmd, "dism /Apply-Ffu /ApplyDrive:%s /ImageFile:\"%s\"", physical, image_path); @@ -1848,8 +1868,7 @@ DWORD WINAPI FormatThread(void* param) // All good } else if (target_type == TT_UEFI) { // For once, no need to do anything - just check our sanity - assert((boot_type == BT_IMAGE) && IS_EFI_BOOTABLE(img_report) && (fs_type <= FS_NTFS)); - if ( (boot_type != BT_IMAGE) || !IS_EFI_BOOTABLE(img_report) || (fs_type > FS_NTFS) ) { + if_not_assert((boot_type == BT_IMAGE) && IS_EFI_BOOTABLE(img_report) && (fs_type <= FS_NTFS)) { ErrorStatus = RUFUS_ERROR(ERROR_INSTALL_FAILURE); goto out; } @@ -1924,7 +1943,8 @@ DWORD WINAPI FormatThread(void* param) ErrorStatus = RUFUS_ERROR(APPERR(ERROR_CANT_PATCH)); } } else { - assert(!img_report.is_windows_img); + if_not_assert(!img_report.is_windows_img) + goto out; if (!ExtractISO(image_path, drive_name, FALSE)) { if (!IS_ERROR(ErrorStatus)) ErrorStatus = RUFUS_ERROR(APPERR(ERROR_ISO_EXTRACT)); diff --git a/src/iso.c b/src/iso.c index a57f22ce341..a5b978be362 100644 --- a/src/iso.c +++ b/src/iso.c @@ -1571,8 +1571,8 @@ uint32_t ReadISOFileToBuffer(const char* iso, const char* iso_file, uint8_t** bu goto out; } file_length = udf_get_file_length(p_udf_file); - if (file_length > UINT32_MAX) { - uprintf("Only files smaller than 4 GB are supported"); + if (file_length > 1 * GB) { + uprintf("Only files smaller than 1 GB are supported"); goto out; } nblocks = (uint32_t)((file_length + UDF_BLOCKSIZE - 1) / UDF_BLOCKSIZE); @@ -1604,10 +1604,11 @@ uint32_t ReadISOFileToBuffer(const char* iso, const char* iso_file, uint8_t** bu goto out; } file_length = p_statbuf->total_size; - if (file_length > UINT32_MAX) { - uprintf("Only files smaller than 4 GB are supported"); + if (file_length > 1 * GB) { + uprintf("Only files smaller than 1 GB are supported"); goto out; } + // coverity[cast_overflow] nblocks = (uint32_t)((file_length + ISO_BLOCKSIZE - 1) / ISO_BLOCKSIZE); *buf = malloc(nblocks * ISO_BLOCKSIZE + 1); if (*buf == NULL) { diff --git a/src/process.c b/src/process.c index 0060c3975f6..5505912e01c 100644 --- a/src/process.c +++ b/src/process.c @@ -407,7 +407,8 @@ static PWSTR GetProcessCommandLine(HANDLE hProcess) ucmdline = (UNICODE_STRING*)(pp + cmd_offset); // In the absolute, someone could craft a process with dodgy attributes to try to cause an overflow - ucmdline->Length = min(ucmdline->Length, 512); + // coverity[cast_overflow] + ucmdline->Length = min(ucmdline->Length, (USHORT)512); wcmdline = (PWSTR)calloc(ucmdline->Length + 1, sizeof(WCHAR)); if (!ReadProcessMemory(hProcess, ucmdline->Buffer, wcmdline, ucmdline->Length, NULL)) { safe_free(wcmdline); @@ -493,8 +494,7 @@ static DWORD WINAPI SearchProcessThread(LPVOID param) // Work on our own copy of the handle names so we don't have to hold the // mutex for string comparison. Update only if the version has changed. if (blocking_process.nVersion[0] != blocking_process.nVersion[1]) { - assert(blocking_process.wHandleName != NULL && blocking_process.nHandles != 0); - if (blocking_process.wHandleName == NULL || blocking_process.nHandles == 0) { + if_not_assert(blocking_process.wHandleName != NULL && blocking_process.nHandles != 0) { ReleaseMutex(hLock); goto out; } @@ -930,8 +930,7 @@ BYTE GetProcessSearch(uint32_t timeout, uint8_t access_mask, BOOL bIgnoreStalePr return 0; } - assert(blocking_process.hLock != NULL); - if (blocking_process.hLock == NULL) + if_not_assert(blocking_process.hLock != NULL) return 0; retry: diff --git a/src/registry.h b/src/registry.h index db85ee3e7c7..e05f00ad38f 100644 --- a/src/registry.h +++ b/src/registry.h @@ -37,11 +37,9 @@ static __inline BOOL DeleteRegistryKey(HKEY key_root, const char* key_name) HKEY hSoftware = NULL; LONG s; - assert(key_root == REGKEY_HKCU); - if (key_root != REGKEY_HKCU) + if_not_assert(key_root == REGKEY_HKCU) return FALSE; - assert(key_name != NULL); - if (key_name == NULL) + if_not_assert(key_name != NULL) return FALSE; if (RegOpenKeyExA(key_root, "SOFTWARE", 0, KEY_READ|KEY_CREATE_SUB_KEY, &hSoftware) != ERROR_SUCCESS) @@ -135,15 +133,12 @@ static __inline BOOL _SetRegistryKey(HKEY key_root, const char* key_name, DWORD HKEY hRoot = NULL, hApp = NULL; DWORD dwDisp, dwType = reg_type; - assert(key_name != NULL); - if (key_name == NULL) + if_not_assert(key_name != NULL) return FALSE; - assert(key_root == REGKEY_HKCU); - if (key_root != REGKEY_HKCU) + if_not_assert(key_root == REGKEY_HKCU) return FALSE; // Validate that we are always dealing with a short key - assert(strchr(key_name, '\\') == NULL); - if (strchr(key_name, '\\') != NULL) + if_not_assert(strchr(key_name, '\\') == NULL) return FALSE; if (RegOpenKeyExA(key_root, NULL, 0, KEY_READ|KEY_CREATE_SUB_KEY, &hRoot) != ERROR_SUCCESS) { diff --git a/src/rufus.c b/src/rufus.c index 74c73cbc208..37747e2548d 100755 --- a/src/rufus.c +++ b/src/rufus.c @@ -1131,7 +1131,8 @@ static void DisplayISOProps(void) // Insert the image name into the Boot selection dropdown and (re)populate the Image option dropdown static void UpdateImage(BOOL update_image_option_only) { - assert(image_index != 0); + if_not_assert(image_index != 0) + return; if (!update_image_option_only) { if (ComboBox_GetItemData(hBootType, image_index) == BT_IMAGE) @@ -1429,8 +1430,7 @@ static DWORD WINAPI BootCheckThread(LPVOID param) } if (boot_type == BT_IMAGE) { - assert(image_path != NULL); - if (image_path == NULL) + if_not_assert(image_path != NULL) goto out; if ((size_check) && (img_report.projected_size > (uint64_t)SelectedDrive.DiskSize)) { // This ISO image is too big for the selected target diff --git a/src/rufus.h b/src/rufus.h index 5007f9bd7d3..471de019667 100644 --- a/src/rufus.h +++ b/src/rufus.h @@ -15,6 +15,7 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ +#include #include #include #include @@ -36,6 +37,7 @@ #define MB 1048576LL #define GB 1073741824LL #define TB 1099511627776LL +#define PB 1125899906842624LL /* * Features not ready for prime time and that may *DESTROY* your data - USE AT YOUR OWN RISKS! @@ -186,6 +188,7 @@ static __inline void static_repchr(char* p, char s, char r) { } #define to_unix_path(str) static_repchr(str, '\\', '/') #define to_windows_path(str) static_repchr(str, '/', '\\') +#define if_not_assert(cond) assert(cond); if (!(cond)) extern void uprintf(const char *format, ...); extern void uprintfs(const char *str); diff --git a/src/rufus.rc b/src/rufus.rc index 8392db42f09..c38e7663179 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2189" +CAPTION "Rufus 4.6.2190" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2189,0 - PRODUCTVERSION 4,6,2189,0 + FILEVERSION 4,6,2190,0 + PRODUCTVERSION 4,6,2190,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2189" + VALUE "FileVersion", "4.6.2190" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2189" + VALUE "ProductVersion", "4.6.2190" END END BLOCK "VarFileInfo" diff --git a/src/stdfn.c b/src/stdfn.c index 2dd576329ee..5265bf94d7d 100644 --- a/src/stdfn.c +++ b/src/stdfn.c @@ -80,8 +80,7 @@ BOOL htab_create(uint32_t nel, htab_table* htab) if (htab == NULL) { return FALSE; } - assert(htab->table == NULL); - if (htab->table != NULL) { + if_not_assert(htab->table == NULL) { uprintf("Warning: htab_create() was called with a non empty table"); return FALSE; } @@ -197,8 +196,7 @@ uint32_t htab_hash(char* str, htab_table* htab) // Not found => New entry // If the table is full return an error - assert(htab->filled < htab->size); - if (htab->filled >= htab->size) { + if_not_assert(htab->filled < htab->size) { uprintf("Hash table is full (%d entries)", htab->size); return 0; } @@ -1199,7 +1197,8 @@ BOOL MountRegistryHive(const HKEY key, const char* pszHiveName, const char* pszH LSTATUS status; HANDLE token = INVALID_HANDLE_VALUE; - assert((key == HKEY_LOCAL_MACHINE) || (key == HKEY_USERS)); + if_not_assert((key == HKEY_LOCAL_MACHINE) || (key == HKEY_USERS)) + return FALSE; if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &token)) { uprintf("Could not get current process token: %s", WindowsErrorString()); @@ -1230,7 +1229,8 @@ BOOL UnmountRegistryHive(const HKEY key, const char* pszHiveName) { LSTATUS status; - assert((key == HKEY_LOCAL_MACHINE) || (key == HKEY_USERS)); + if_not_assert((key == HKEY_LOCAL_MACHINE) || (key == HKEY_USERS)) + return FALSE; status = RegUnLoadKeyA(key, pszHiveName); if (status != ERROR_SUCCESS) { diff --git a/src/stdio.c b/src/stdio.c index 61984aff8d4..59909196c37 100644 --- a/src/stdio.c +++ b/src/stdio.c @@ -857,7 +857,8 @@ uint32_t ResolveDllAddress(dll_resolver_t* resolver) // NB: SymLoadModuleEx() does not load a PDB unless the file has an explicit '.pdb' extension base_address = pfSymLoadModuleEx(hRufus, NULL, path, NULL, DEFAULT_BASE_ADDRESS, 0, NULL, 0); - assert(base_address == DEFAULT_BASE_ADDRESS); + if_not_assert(base_address == DEFAULT_BASE_ADDRESS) + goto out; // On Windows 11 ARM64 the following call will return *TWO* different addresses for the same // call, because most Windows DLL's are ARM64X, which means that they are an unholy union of // both X64 and ARM64 code in the same binary... diff --git a/src/stdlg.c b/src/stdlg.c index e75987c424f..39e0e127f14 100644 --- a/src/stdlg.c +++ b/src/stdlg.c @@ -588,7 +588,8 @@ INT_PTR CALLBACK NotificationCallback(HWND hDlg, UINT message, WPARAM wParam, LP return (INT_PTR)TRUE; case IDC_MORE_INFO: if (notification_more_info != NULL) { - assert(notification_more_info->callback != NULL); + if_not_assert(notification_more_info->callback != NULL) + return (INT_PTR)FALSE; if (notification_more_info->id == MORE_INFO_URL) { ShellExecuteA(hDlg, "open", notification_more_info->url, NULL, NULL, SW_SHOWNORMAL); } else { diff --git a/src/ui.c b/src/ui.c index 166e2b0abae..e83c77a35cc 100644 --- a/src/ui.c +++ b/src/ui.c @@ -575,7 +575,8 @@ void SetSectionHeaders(HWND hDlg) memset(wtmp, 0, sizeof(wtmp)); GetWindowTextW(hCtrl, wtmp, ARRAYSIZE(wtmp) - 4); wlen = wcslen(wtmp); - assert(wlen < ARRAYSIZE(wtmp) - 2); + if_not_assert(wlen < ARRAYSIZE(wtmp) - 2) + break; wtmp[wlen++] = L' '; wtmp[wlen++] = L' '; SetWindowTextW(hCtrl, wtmp); diff --git a/src/vhd.c b/src/vhd.c index 4ab0236da0d..4446d89ad49 100644 --- a/src/vhd.c +++ b/src/vhd.c @@ -1006,8 +1006,9 @@ static DWORD WINAPI VhdSaveImageThread(void* param) OVERLAPPED overlapped = { 0 }; DWORD r = ERROR_NOT_FOUND, flags; - assert(img_save->Type == VIRTUAL_STORAGE_TYPE_DEVICE_VHD || - img_save->Type == VIRTUAL_STORAGE_TYPE_DEVICE_VHDX); + if_not_assert(img_save->Type == VIRTUAL_STORAGE_TYPE_DEVICE_VHD || + img_save->Type == VIRTUAL_STORAGE_TYPE_DEVICE_VHDX) + return ERROR_INVALID_PARAMETER; UpdateProgressWithInfoInit(NULL, FALSE); diff --git a/src/wue.c b/src/wue.c index b257cb67cd9..9fadd53d9b2 100644 --- a/src/wue.c +++ b/src/wue.c @@ -905,7 +905,8 @@ BOOL ApplyWindowsCustomization(char drive_letter, int flags) // If we have a windowsPE section, copy the answer files to the root of boot.wim as // Autounattend.xml. This also results in that file being automatically copied over // to %WINDIR%\Panther\unattend.xml for later passes processing. - assert(mount_path != NULL); + if_not_assert(mount_path != NULL) + goto out; static_sprintf(path, "%s\\Autounattend.xml", mount_path); if (!CopyFileU(unattend_xml_path, path, TRUE)) { uprintf("Could not create boot.wim 'Autounattend.xml': %s", WindowsErrorString()); From 20881ceea6f4e941dc4441a448866fb8292f5ef8 Mon Sep 17 00:00:00 2001 From: Dmytro Hissa Date: Thu, 25 Jul 2024 13:32:11 +0100 Subject: [PATCH 09/30] [loc] fix the translation of 'cancel' for Ukrainian --- res/loc/po/uk-UA.po | 14 +++++++------- res/loc/rufus.loc | 10 +++++----- src/rufus.rc | 10 +++++----- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/res/loc/po/uk-UA.po b/res/loc/po/uk-UA.po index 39a79651284..dc93bf4bf0f 100644 --- a/res/loc/po/uk-UA.po +++ b/res/loc/po/uk-UA.po @@ -2,8 +2,8 @@ msgid "" msgstr "" "Project-Id-Version: 4.5\n" "Report-Msgid-Bugs-To: pete@akeo.ie\n" -"POT-Creation-Date: 2024-05-14 13:59+0300\n" -"PO-Revision-Date: 2024-05-15 21:15+0300\n" +"POT-Creation-Date: 2024-07-23 23:38+0300\n" +"PO-Revision-Date: 2024-07-23 23:42+0300\n" "Last-Translator: \n" "Language-Team: \n" "Language: uk_UA\n" @@ -207,7 +207,7 @@ msgid "" "To continue with this operation, click OK. To quit click CANCEL." msgstr "" "УВАГА: ВСІ ДАНІ НА ДИСКУ '%s' БУДУТЬ ЗНИЩЕНІ.\n" -"Для того, щоб продовжити, натисніть ОК. Для виходу натисніть Відмінити." +"Для того, щоб продовжити, натисніть ОК. Для виходу натисніть Скасувати." #. • MSG_004 msgid "Rufus update policy" @@ -219,7 +219,7 @@ msgstr "Чи хочете ви, щоб Rufus перевіряв оновленн #. • MSG_007 msgid "Cancel" -msgstr "Відмінити" +msgstr "Скасувати" #. • MSG_010 msgid "Bad blocks found" @@ -365,7 +365,7 @@ msgstr "Додаток" #. • MSG_038 msgid "Abort" -msgstr "Відмінити" +msgstr "Скасувати" #. • MSG_039 msgid "Launch" @@ -768,7 +768,7 @@ msgstr "" "\n" "Так як нові версії Syslinux несумісні одна з одною, необхідно, щоб додаткові файли ('ldlinux.sys' та 'ldlinux.bss') були завантажені з інтернету:\n" "- Виберіть 'Так', щоб завантажити файли з інтернету\n" -"- Виберіть 'Ні', щоб відмінити операцію\n" +"- Виберіть 'Ні', щоб скасувати операцію\n" "\n" "Примітка: Файли будуть завантажені в поточний каталог додатку і будуть використовуватися повторно за необхідності." @@ -795,7 +795,7 @@ msgstr "" "Різні версії завантажувача Grub можуть бути несумісними одна з одною та увімкнення їх усіх не являється можливим. Rufus намагатиметься знайти потрібну версію настановного файла завантажувача Grub ('core.img'), що відповідає версії завантажувача вашого образу:\n" "- Виберіть 'Так', щоб скачати завантажувальний файл з інтернету\n" "- Виберіть 'Ні', щоб використовувати версію за замовчуванням із Rufus\n" -"- Виберіть 'Відмінити', щоб відмінити операцію\n" +"- Виберіть 'Скасувати', щоб скасувати операцію\n" "\n" "Примітка: Файл буде завантажено в поточний каталог додатку та буде використаний повторно наступного разу. Якщо потрібна версія настановного файла не буде знайдена в інтернеті, то буде використовуватися версія за замовчуванням." diff --git a/res/loc/rufus.loc b/res/loc/rufus.loc index 7c9297e8906..2b562a4bcbb 100644 --- a/res/loc/rufus.loc +++ b/res/loc/rufus.loc @@ -15327,11 +15327,11 @@ t IDC_CHECK_NOW "Перевірити зараз" t MSG_001 "Виявлено інший екземпляр Rufus" t MSG_002 "Запущено інший додаток Rufus.\nЗакрийте перший додаток перед тим, як запустити ще один." -t MSG_003 "УВАГА: ВСІ ДАНІ НА ДИСКУ '%s' БУДУТЬ ЗНИЩЕНІ.\nДля того, щоб продовжити, натисніть ОК. Для виходу натисніть Відмінити." +t MSG_003 "УВАГА: ВСІ ДАНІ НА ДИСКУ '%s' БУДУТЬ ЗНИЩЕНІ.\nДля того, щоб продовжити, натисніть ОК. Для виходу натисніть Скасувати." t MSG_004 "Політика оновлення Rufus" t MSG_005 "Чи хочете ви, щоб Rufus перевіряв оновлення автоматично?" t MSG_006 "Закрити" -t MSG_007 "Відмінити" +t MSG_007 "Скасувати" t MSG_008 "Так" t MSG_009 "Ні" t MSG_010 "Знайдено пошкоджені блоки" @@ -15362,7 +15362,7 @@ t MSG_034 "%d прохід" t MSG_035 "%d проходів %s" t MSG_036 "ISO-образ" t MSG_037 "Додаток" -t MSG_038 "Відмінити" +t MSG_038 "Скасувати" t MSG_039 "Запустити" t MSG_040 "Завантажити" t MSG_041 "Операція скасована користувачем" @@ -15438,9 +15438,9 @@ t MSG_110 "MS-DOS не може завантажитися при викорис t MSG_111 "Несумісний розмір кластера" t MSG_112 "Форматування великих UDF-томів займає багато часу. На швидкості USB 2.0 приблизна тривалість форматування складе %d:%02d, протягом якої індикатор прогресу буде заморожено. Будь ласка, будьте терплячі!" t MSG_113 "Великий UDF-том" -t MSG_114 "Даний образ використовує Syslinux %s%s, проте даний додаток включає в себе тільки настановні файли для Syslinux %s%s.\n\nТак як нові версії Syslinux несумісні одна з одною, необхідно, щоб додаткові файли ('ldlinux.sys' та 'ldlinux.bss') були завантажені з інтернету:\n- Виберіть 'Так', щоб завантажити файли з інтернету\n- Виберіть 'Ні', щоб відмінити операцію\n\nПримітка: Файли будуть завантажені в поточний каталог додатку і будуть використовуватися повторно за необхідності." +t MSG_114 "Даний образ використовує Syslinux %s%s, проте даний додаток включає в себе тільки настановні файли для Syslinux %s%s.\n\nТак як нові версії Syslinux несумісні одна з одною, необхідно, щоб додаткові файли ('ldlinux.sys' та 'ldlinux.bss') були завантажені з інтернету:\n- Виберіть 'Так', щоб завантажити файли з інтернету\n- Виберіть 'Ні', щоб скасувати операцію\n\nПримітка: Файли будуть завантажені в поточний каталог додатку і будуть використовуватися повторно за необхідності." t MSG_115 "Необхідно завантажити" -t MSG_116 "Даний образ використовує завантажувач Grub %s, але додаток включає тільки настановні файли для завантажувача Grub %s.\n\nРізні версії завантажувача Grub можуть бути несумісними одна з одною та увімкнення їх усіх не являється можливим. Rufus намагатиметься знайти потрібну версію настановного файла завантажувача Grub ('core.img'), що відповідає версії завантажувача вашого образу:\n- Виберіть 'Так', щоб скачати завантажувальний файл з інтернету\n- Виберіть 'Ні', щоб використовувати версію за замовчуванням із Rufus\n- Виберіть 'Відмінити', щоб відмінити операцію\n\nПримітка: Файл буде завантажено в поточний каталог додатку та буде використаний повторно наступного разу. Якщо потрібна версія настановного файла не буде знайдена в інтернеті, то буде використовуватися версія за замовчуванням." +t MSG_116 "Даний образ використовує завантажувач Grub %s, але додаток включає тільки настановні файли для завантажувача Grub %s.\n\nРізні версії завантажувача Grub можуть бути несумісними одна з одною та увімкнення їх усіх не являється можливим. Rufus намагатиметься знайти потрібну версію настановного файла завантажувача Grub ('core.img'), що відповідає версії завантажувача вашого образу:\n- Виберіть 'Так', щоб скачати завантажувальний файл з інтернету\n- Виберіть 'Ні', щоб використовувати версію за замовчуванням із Rufus\n- Виберіть 'Скасувати', щоб скасувати операцію\n\nПримітка: Файл буде завантажено в поточний каталог додатку та буде використаний повторно наступного разу. Якщо потрібна версія настановного файла не буде знайдена в інтернеті, то буде використовуватися версія за замовчуванням." t MSG_117 "Стандартне встановлення Windows" t MSG_119 "розширені властивості диска" t MSG_120 "розширені властивості форматування" diff --git a/src/rufus.rc b/src/rufus.rc index c38e7663179..2696caf115d 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2190" +CAPTION "Rufus 4.6.2191" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2190,0 - PRODUCTVERSION 4,6,2190,0 + FILEVERSION 4,6,2191,0 + PRODUCTVERSION 4,6,2191,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2190" + VALUE "FileVersion", "4.6.2191" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2190" + VALUE "ProductVersion", "4.6.2191" END END BLOCK "VarFileInfo" From 07d6e07694da0f9e5910517077f4c8a99e06d1b6 Mon Sep 17 00:00:00 2001 From: Maksim Bondarenkov <119937608+ognevny@users.noreply.github.com> Date: Tue, 13 Aug 2024 20:40:39 +0300 Subject: [PATCH 10/30] [iso] fix MSVC-only definition * It should be defined for MSVC only. This fixes UCRT build. * Closes #2540. --- src/libcdio/config.h | 3 +++ src/rufus.rc | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libcdio/config.h b/src/libcdio/config.h index b0ee3c991ca..a782044fadc 100644 --- a/src/libcdio/config.h +++ b/src/libcdio/config.h @@ -39,8 +39,11 @@ /* Define to 1 if you have the `fseeko64' function. */ #define HAVE_FSEEKO64 1 + +#if defined(_MSC_VER) /* The equivalent of fseeko64 for MSVC is _fseeki64 */ #define fseeko64 _fseeki64 +#endif /* Define to 1 if you have the `ftruncate' function. */ /* #undef HAVE_FTRUNCATE */ diff --git a/src/rufus.rc b/src/rufus.rc index 2696caf115d..7d45ba7649b 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2191" +CAPTION "Rufus 4.6.2192" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2191,0 - PRODUCTVERSION 4,6,2191,0 + FILEVERSION 4,6,2192,0 + PRODUCTVERSION 4,6,2192,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2191" + VALUE "FileVersion", "4.6.2192" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2191" + VALUE "ProductVersion", "4.6.2192" END END BLOCK "VarFileInfo" From 0ee7d7a2c033bb3a245c2a78be7ca81b51fd4baf Mon Sep 17 00:00:00 2001 From: Jakub Zieciak Date: Sat, 17 Aug 2024 16:13:30 +0100 Subject: [PATCH 11/30] [loc] update Polish translation --- res/loc/po/pl-PL.po | 64 ++++++++++++++++++++++----------------------- res/loc/rufus.loc | 52 ++++++++++++++++++------------------ src/rufus.rc | 10 +++---- 3 files changed, 63 insertions(+), 63 deletions(-) diff --git a/res/loc/po/pl-PL.po b/res/loc/po/pl-PL.po index 3d91def047e..303214be523 100644 --- a/res/loc/po/pl-PL.po +++ b/res/loc/po/pl-PL.po @@ -2,8 +2,8 @@ msgid "" msgstr "" "Project-Id-Version: 4.5\n" "Report-Msgid-Bugs-To: pete@akeo.ie\n" -"POT-Creation-Date: 2024-05-09 19:08+0100\n" -"PO-Revision-Date: 2024-05-09 19:38+0100\n" +"POT-Creation-Date: 2024-08-13 12:38+0100\n" +"PO-Revision-Date: 2024-08-13 13:22+0100\n" "Last-Translator: \n" "Language-Team: \n" "Language: pl_PL\n" @@ -173,7 +173,7 @@ msgstr "Szukaj aktualizacji - Rufus" #. • IDD_NEW_VERSION → IDS_NEW_VERSION_AVAIL_TXT msgid "A newer version is available. Please download the latest version!" -msgstr "Jest dostępna nowsza wersja. Proszę pobrać najnowszą wersję!" +msgstr "Nowsza wersja jest dostępna. Proszę pobrać najnowszą wersję!" #. • IDD_NEW_VERSION → IDC_WEBSITE msgid "Click here to go to the website" @@ -491,7 +491,7 @@ msgstr "Nie można skopiować plików na dysk docelowy." #. • MSG_070 msgid "Cancelled by user." -msgstr "Przerwane przez użytkwonika." +msgstr "Przerwane przez użytkownika." #. • MSG_071 #. @@ -517,7 +517,7 @@ msgstr "Nie można ponownie zamontować woluminu." #. • MSG_076 msgid "Unable to patch/setup files for boot." -msgstr "Nie można załatać/skonfigurować plików do bootowania." +msgstr "Nie można załatać/skonfigurować plików do rozruchu." #. • MSG_077 msgid "Unable to assign a drive letter." @@ -551,7 +551,7 @@ msgstr "Niewspierany obraz" #. • MSG_082 msgid "This image is either non-bootable, or it uses a boot or compression method that is not supported by Rufus..." -msgstr "Ten obraz nie jest bootowalny lub używa kompresji sektora rozruchowego niewspieranej przez Rufusa..." +msgstr "Ten obraz nie jest rozruchowy lub używa kompresji sektora rozruchowego niewspieranej przez Rufusa..." #. • MSG_083 msgid "Replace %s?" @@ -612,7 +612,7 @@ msgstr "Niewspierane ISO" #. • MSG_091 msgid "When using UEFI Target Type, only EFI bootable ISO images are supported. Please select an EFI bootable ISO or set the Target Type to BIOS." -msgstr "Podczas używania Systemu Typu UEFI, tylko bootowalne obrazy ISO EFI są wspierane. Proszę wybrać bootowalny obraz ISO EFI lub ustawić System Docelowy na BIOS." +msgstr "Podczas używania Systemu Typu UEFI, tylko rozruchowe obrazy ISO EFI są wspierane. Proszę wybrać rozruchowy obraz ISO EFI lub ustawić System Docelowy na BIOS." #. • MSG_092 msgid "Unsupported filesystem" @@ -652,7 +652,7 @@ msgid "" "\n" "Note: The 'FIXED/REMOVABLE' attribute is a hardware property that can only be changed using custom tools from the drive manufacturer. However those tools are ALMOST NEVER provided to the public..." msgstr "" -"WAŻNE: Próbujesz zainstalować 'Windows To Go', lecz dysk docelowy nie ma atrybytu 'LOKALNY'. Z tego powodu Windows prawdopodobnie zawiesi się podczas startu, ponieważ Microsoft nie zaprojektował go do działania z dyskami, które zamiast tego mają atrybut 'WYMIENNY'.\n" +"WAŻNE: Próbujesz zainstalować 'Windows To Go', lecz dysk docelowy nie ma atrybutu 'LOKALNY'. Z tego powodu Windows prawdopodobnie zawiesi się podczas startu, ponieważ Microsoft nie zaprojektował go do działania z dyskami, które zamiast tego mają atrybut 'WYMIENNY'.\n" "\n" "Czy nadal chcesz kontynuować?\n" "\n" @@ -675,7 +675,7 @@ msgid "" "Your platform cannot extract files from WIM archives. WIM extraction is required to create EFI bootable Windows 7 and Windows Vista USB drives. You can fix that by installing a recent version of 7-Zip.\n" "Do you want to visit the 7-zip download page?" msgstr "" -"Twoja platforma nie może wypakować plików z archiwów WIM. Wypakowanie WIM jest wymagane aby utworzyć bootowalny dysk USB EFI z Windows 7 i Windows Vista. Możesz to naprawić instalując aktualną wersję 7-Zip.\n" +"Twoja platforma nie może wypakować plików z archiwów WIM. Wypakowanie WIM jest wymagane aby utworzyć rozruchowy dysk USB EFI z Windows 7 i Windows Vista. Możesz to naprawić instalując aktualną wersję 7-Zip.\n" "Czy chcesz odwiedzić stronę pobierania 7-zip?" #. • MSG_103 @@ -700,7 +700,7 @@ msgstr "" "Ponieważ ten plik jest większy niż 100 KB i zawsze jest obecny na obrazach ISO %s, nie jest dołączony do Rufusa.\n" "\n" "Rufus może pobrać brakujący plik dla ciebie:\n" -"- Wybierz 'Yes', aby połączyć sie z internetem i pobrać plik\n" +"- Wybierz 'Yes', aby połączyć się z internetem i pobrać plik\n" "- Wybierz 'No', jeśli chcesz później ręcznie skopiować ten plik na dysk \n" "\n" "Informacja: Plik zostanie pobrany do obecnego folderu i jeśli '%s' istnieje, zostanie użyty ponownie automatycznie." @@ -710,7 +710,7 @@ msgid "" "Cancelling may leave the device in an UNUSABLE state.\n" "If you are sure you want to cancel, click YES. Otherwise, click NO." msgstr "" -"Przerwanie operacji może pozostawic urządzenie w stanie NIEZDATNYM DO UŻYTKU.\n" +"Przerwanie operacji może pozostawić urządzenie w stanie NIEZDATNYM DO UŻYTKU.\n" "Jeśli jesteś pewien, że chcesz przerwać operację, naciśnij TAK. W innym razie naciśnij NIE." #. • MSG_106 @@ -860,7 +860,7 @@ msgstr "Nie pokazuj więcej tej wiadomości" #. • MSG_128 msgid "Important notice about %s" -msgstr "Ważna informacjoa o %s" +msgstr "Ważna informacja o %s" #. • MSG_129 msgid "" @@ -896,7 +896,7 @@ msgid "" msgstr "" "Rufus wykrył, że próbujesz utworzyć nośnik \"Windows To Go\" bazujący na ISO 1809.\n" "\n" -"Z powodu *BŁĘDU MICROSOFTU*, ten nośnik ulegnie awarii podczas bootowania systemu (Blue Screen of Death), chyba że ręcznie zastąpisz plik \"WppRecorder.sys\" wersją z 1803.\n" +"Z powodu *BŁĘDU MICROSOFTU*, ten nośnik ulegnie awarii podczas rozruchu systemu (Blue Screen of Death), chyba że ręcznie zastąpisz plik \"WppRecorder.sys\" wersją z 1803.\n" "\n" "Należy również pamiętać, że powodem, dla którego Rufus nie może automatycznie rozwiązać tego problemu, jest to, że \"WppRecorder.sys” jest plikiem chronionym prawem autorskim firmy Microsoft, więc nie możemy zgodnie z prawem osadzić kopii pliku w aplikacji." @@ -968,7 +968,7 @@ msgstr "Pobierz obraz ISO" #. • MSG_150 msgid "Type of computer you plan to use this bootable drive with. It is your responsibility to determine whether your target is of BIOS or UEFI type before you start creating the drive, as it may fail to boot otherwise." -msgstr "Typ komputera, na którym chcesz użyć bootowalnego dysku. Przed stworzeniem dysku musisz sam określić, czy docelowy komputer posiada BIOS, czy UEFI, inaczej może się nie uruchomić." +msgstr "Typ komputera, na którym chcesz użyć dysku rozruchowego. Przed stworzeniem dysku musisz sam określić, czy docelowy komputer posiada BIOS, czy UEFI, inaczej może się nie uruchomić." #. • MSG_151 #. @@ -1018,7 +1018,7 @@ msgstr "Przełącz opcje zaawansowane" #. • MSG_161 msgid "Check the device for bad blocks using a test pattern" -msgstr "Sprawdź urządzenie pod kątem błędnych sektorów używając wzoru testoego" +msgstr "Sprawdź urządzenie pod kątem błędnych sektorów używając wzoru testowego" #. • MSG_162 msgid "Uncheck this box to use the \"slow\" format method" @@ -1030,7 +1030,7 @@ msgstr "Metoda, która zostanie użyta do stworzenia partycji" #. • MSG_164 msgid "Method that will be used to make the drive bootable" -msgstr "Metoda, która zostanie użyta, aby uczynić dysk bootowalnym" +msgstr "Metoda, która zostanie użyta, aby uczynić dysk rozruchowym" #. • MSG_165 msgid "Click to select or download an image..." @@ -1128,7 +1128,7 @@ msgid "" "Rufus does not install or run background services, therefore update checks are performed only when the main application is running.\\line\n" "Internet access is of course required when checking for updates." msgstr "" -"Rufus nie instaluje lub uruchamia procesów w tle, dlatego aktualizacje są sprawdzane wylącznie przy starcie głównego okna aplikacji.\\line\n" +"Rufus nie instaluje lub uruchamia procesów w tle, dlatego aktualizacje są sprawdzane wyłącznie przy starcie głównego okna aplikacji.\\line\n" "Dostęp do internetu jest wymagany podczas sprawdzania aktualizacji." #. • MSG_187 @@ -1182,7 +1182,7 @@ msgid "" msgstr "" "WAŻNE: TEN DYSK UŻYWA NIESTANDARDOWEGO ROZMIARU SEKTORA!\n" "\n" -"Standardowe dyski używają 512-bajtowego rozmiaru sektora, lecz ten dysk używa %d-bajtowego. W wielu przypadkach oznacza to, że NIE będziesz mógł bootować z tego dysku.\n" +"Standardowe dyski używają 512-bajtowego rozmiaru sektora, lecz ten dysk używa %d-bajtowego. W wielu przypadkach oznacza to, że NIE będziesz mógł startować z tego dysku.\n" "Rufus może spróbować utworzyć dysk rozruchowy, ale NIE MA GWARANCJI, że to zadziała." #. • MSG_197 @@ -1350,7 +1350,7 @@ msgstr "Kopiowanie plików ISO: %s" #. • MSG_232 msgid "Win7 EFI boot setup (%s)..." -msgstr "Ustawianie bootowania Win7 EFI (%s)..." +msgstr "Ustawianie rozruchu Win7 EFI (%s)..." #. • MSG_233 msgid "Finalizing, please wait..." @@ -1552,9 +1552,9 @@ msgid "" "\n" "Please select the mode that you want to use to write this image:" msgstr "" -"Obraz, który wybrałeś to obraz 'ISOHybrid'. To znacza iż może zostać zapisany w trybie %s (kopia plików) lub trybie %s (obraz dysku).\n" +"Obraz, który wybrałeś to obraz 'ISOHybrid'. Oznacza to, że może zostać zapisany w trybie %s (kopia plików) lub trybie %s (obraz dysku).\n" "Rufus zaleca użycie trybu %s, abyś zawsze miał pełny dostęp do dysku po jego zapisaniu.\n" -"Jednak jeśli napotkasz problemy podczas bootowania możesz spróbować zapisać ten obraz ponownie w trybie %s.\n" +"Jednak jeśli napotkasz problemy podczas rozruchu możesz spróbować zapisać ten obraz ponownie w trybie %s.\n" "\n" "Proszę wybrać tryb którego chcesz użyć do zapisania tego obrazu:" @@ -1739,13 +1739,13 @@ msgid "" "\n" "Please select the mode that you want to use to write this image:" msgstr "" -"Obraz ISO, który został wybrany używa UEFI i jest dość mały, żeby być zapisanym jako partycja systemowa EFI (ESP). Zapisywanie do ESP zamiast to ogólnej partycji zajmującej cały dysk. Może być bardziej odpowiedni dla niektórych typów instalacji.\n" +"Wybrany plik ISO używa UEFI i jest na tyle mały, że można go zapisać jako partycję systemową EFI (ESP). Zapisanie na ESP, zamiast zapisywania na ogólnej partycji danych zajmującej cały dysk, może być preferowane dla niektórych typów instalacji.\n" "\n" -"Proszę wybrać tryb którym chesz zapisać ten obraz:" +"Proszę wybrać tryb, w którym chcesz zapisać ten obraz:" #. • MSG_311 msgid "Use %s (in the main application window) to enable." -msgstr "Użyj %s w głownym oknie aplikacji, żeby włączyć." +msgstr "Użyj %s w głównym oknie aplikacji, żeby włączyć." #. • MSG_312 msgid "Extra hashes (SHA512)" @@ -1853,9 +1853,9 @@ msgid "" msgstr "" "Dodatkowy plik (\"diskcopy.dll\") musi zostać pobrany od Microsoft, żeby zainstalować MS-DOS:\n" "- Wybierz opcję \"Tak\", aby pobrać pliku z internetu\n" -"- Wybierz opcję \"Nie\", aby anulować operajcę\n" +"- Wybierz opcję \"Nie\", aby anulować operację\n" "\n" -"Uwaga: Plik zostanie pobrany do folderu aplikacji i będzie użyty pownownie, gdy będzie dostępny." +"Uwaga: Plik zostanie pobrany do folderu aplikacji i będzie użyty ponownie, gdy będzie dostępny." #. • MSG_338 msgid "Revoked UEFI bootloader detected" @@ -1868,9 +1868,9 @@ msgid "" "- If you obtained this ISO image from a non reputable source, you should consider the possibility that it might contain UEFI malware and avoid booting from it.\n" "- If you obtained it from a trusted source, you should try to locate a more up to date version, that will not produce this warning." msgstr "" -"Rufus wykrył, że wybrany plik ISO zawiera bootloader UEFI, który został unieważniony, i który wygenereuje %s. gdy włączony zostanie \"Secure Boot\" w pełni zaktualizowanym systemie UEFI.\n" +"Rufus wykrył, że wybrany plik ISO zawiera bootloader UEFI, który został unieważniony, i który wygeneruje %s. gdy włączony zostanie \"Secure Boot\" w pełni zaktualizowanym systemie UEFI.\n" "\n" -"- Jeśli uzyskałeś ten obraz ISO z niezaufanego źródła, powinienneś rozważyć możliwość, że może on zawierać złośliwe oprogramowanie dla UEFI i unikać jego uruchamiania.\n" +"- Jeśli uzyskałeś ten obraz ISO z niezaufanego źródła, powinieneś rozważyć możliwość, że może on zawierać złośliwe oprogramowanie dla UEFI i unikać jego uruchamiania.\n" "- Jeśli uzyskałeś go z zaufanego źródła, powinieneś spróbować znaleźć nowszą wersję, która nie wygeneruje tego ostrzeżenia." #. • MSG_340 @@ -1923,7 +1923,7 @@ msgstr "Użyj Rufus MBR" #. #. The following messages are for the Windows Store listing only and are not used by the application msgid "Rufus is a utility that helps format and create bootable USB flash drives, such as USB keys/pendrives, memory sticks, etc." -msgstr "Rufus to narzędzie, które pomaga formatować i tworzyć bootowalne dyski flash USB, takie jak klucze USB / pendrive, pendrive'y itp." +msgstr "Rufus to narzędzie, które pomaga formatować i tworzyć dyski rozruchowe flash USB, takie jak klucze USB, pendrive-y itp." #. • MSG_901 msgid "Official site: %s" @@ -1962,7 +1962,7 @@ msgstr "Sformatuj nośnik używając: FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3" #. • MSG_911 msgid "Create FreeDOS bootable USB drives" -msgstr "Stwórz bootowalny nośnik USB FreeDOS" +msgstr "Stwórz nośnik rozruchowy USB FreeDOS" #. • MSG_912 msgid "Create bootable drives from bootable ISOs (Windows, Linux, etc.)" @@ -1986,7 +1986,7 @@ msgstr "Twórz dyski instalacyjne systemu Windows 11 dla komputerów, które nie #. • MSG_917 msgid "Create persistent Linux partitions" -msgstr "Stwórz particje persistent Linuxa" +msgstr "Utwórz trwałe partycje Linux" #. • MSG_918 msgid "Create VHD/DD images of the selected drive" @@ -1998,7 +1998,7 @@ msgstr "Oblicz sumy kontrolne MD5, SHA-1, SHA-256 i SHA-512 dla wybranego obrazu #. • MSG_920 msgid "Perform bad blocks checks, including detection of \"fake\" flash drives" -msgstr "Sprawdź dysk pod względem spójności danych lub wykryj \"nieorginalny\" pendrive" +msgstr "Sprawdź dysk pod względem spójności danych lub wykryj \"nieoryginalny\" pendrive" #. • MSG_921 msgid "Download official Microsoft Windows retail ISOs" diff --git a/res/loc/rufus.loc b/res/loc/rufus.loc index 2b562a4bcbb..0849e65b2d0 100644 --- a/res/loc/rufus.loc +++ b/res/loc/rufus.loc @@ -10280,7 +10280,7 @@ t IDC_LOG_SAVE "Zapisz" g IDD_NEW_VERSION t IDCANCEL "Zamknij" t IDD_NEW_VERSION "Szukaj aktualizacji - Rufus" -t IDS_NEW_VERSION_AVAIL_TXT "Jest dostępna nowsza wersja. Proszę pobrać najnowszą wersję!" +t IDS_NEW_VERSION_AVAIL_TXT "Nowsza wersja jest dostępna. Proszę pobrać najnowszą wersję!" t IDC_WEBSITE "Kliknij tutaj, aby przejść na stronę www" t IDS_NEW_VERSION_NOTES_GRP "Informacje o wydaniu" t IDS_NEW_VERSION_DOWNLOAD_GRP "Pobierz" @@ -10363,19 +10363,19 @@ t MSG_066 "Niepowodzenie instalacji" t MSG_067 "Nie można otworzyć nośnika. Może być używany przez inny proces. Proszę podłączyć ponownie nośnik i spróbować jeszcze raz." t MSG_068 "Nie można podzielić dysku na partycje." t MSG_069 "Nie można skopiować plików na dysk docelowy." -t MSG_070 "Przerwane przez użytkwonika." +t MSG_070 "Przerwane przez użytkownika." t MSG_071 "Nie można rozpocząć wątku." t MSG_072 "Nie ukończono sprawdzania błędnych sektorów." t MSG_073 "Niepowodzenie skanowania pliku ISO." t MSG_074 "Niepowodzenie wypakowywania pliku ISO." t MSG_075 "Nie można ponownie zamontować woluminu." -t MSG_076 "Nie można załatać/skonfigurować plików do bootowania." +t MSG_076 "Nie można załatać/skonfigurować plików do rozruchu." t MSG_077 "Nie można przypisać litery dysku." t MSG_078 "Nie można zamontować woluminu GUID." t MSG_079 "Urządzenie nie jest gotowe." t MSG_080 "Rufus wykrył, że Windows wciąż opróżnia swoje bufory na urządzenie USB.\n\nZależnie od prędkości twojego urządzenia USB, ta operacja może zająć dużo czasu, zwłaszcza dla dużych plików.\n\nZalecamy, abyś pozwolił Windowsowi skończyć, żeby uniknąć uszkodzeń. Jeśli jesteś już zmęczony czekaniem, możesz po prostu odłączyć urządzenie..." t MSG_081 "Niewspierany obraz" -t MSG_082 "Ten obraz nie jest bootowalny lub używa kompresji sektora rozruchowego niewspieranej przez Rufusa..." +t MSG_082 "Ten obraz nie jest rozruchowy lub używa kompresji sektora rozruchowego niewspieranej przez Rufusa..." t MSG_083 "Zastąpić %s?" t MSG_084 "Ten obraz ISO wydaje się używać przestarzałej wersji '%s'.\nZ tego powodu menu rozruchowe może nie wyświetlać się poprawnie.\n\nNowsza wersja może być pobrana przez Rufus, aby naprawić ten błąd:\n- Wybierz 'Tak', aby połączyć się z internetem i pobrać plik\n- Wybierz 'Nie', aby zostawić istniejący plik ISO niezmienionym\nJeśli nie wiesz co zrobić, powinieneś wybrać 'Tak'.\n\nInformacja: Nowy plik będzie pobrany do aktualnej lokalizacji i jeśli '%s' istnieje, zostanie ponownie użyte automatycznie." t MSG_085 "Pobieranie %s" @@ -10384,21 +10384,21 @@ t MSG_087 "dla NAND %s" t MSG_088 "Obraz jest zbyt duży" t MSG_089 "Ten obraz jest zbyt duży dla wybranego celu." t MSG_090 "Niewspierane ISO" -t MSG_091 "Podczas używania Systemu Typu UEFI, tylko bootowalne obrazy ISO EFI są wspierane. Proszę wybrać bootowalny obraz ISO EFI lub ustawić System Docelowy na BIOS." +t MSG_091 "Podczas używania Systemu Typu UEFI, tylko rozruchowe obrazy ISO EFI są wspierane. Proszę wybrać rozruchowy obraz ISO EFI lub ustawić System Docelowy na BIOS." t MSG_092 "Niewspierany system plików" t MSG_093 "WAŻNE: TEN NAPĘD ZAWIERA WIELE PARTYCJI!!\n\nMoże to obejmować partycje/woluminy, które nie są wymienione lub nawet widoczne z poziomu Windows. Jeżeli chcesz kontynuować, jesteś odpowiedzialny za utratę danych na tych partycjach." t MSG_094 "Wykryto wiele partycji" t MSG_095 "Obraz DD" t MSG_096 "Aktualnie zaznaczony system plików nie może zostać użyty z tym typem ISO. Proszę wybrać inny system plików lub użyć innego ISO." t MSG_097 "'%s' może zostać użyte wyłącznie, jeżeli system plików to NTFS." -t MSG_098 "WAŻNE: Próbujesz zainstalować 'Windows To Go', lecz dysk docelowy nie ma atrybytu 'LOKALNY'. Z tego powodu Windows prawdopodobnie zawiesi się podczas startu, ponieważ Microsoft nie zaprojektował go do działania z dyskami, które zamiast tego mają atrybut 'WYMIENNY'.\n\nCzy nadal chcesz kontynuować?\n\nInformacja: Atrybut 'LOKALNY/WYMIENNY' to właściwość sprzętowa, która może jedynie zostać zmieniona używając narzędzi dostarczanych przez producenta dysku. Jednak te narzędzia PRAWIE NIGDY nie są udostępniane publicznie..." +t MSG_098 "WAŻNE: Próbujesz zainstalować 'Windows To Go', lecz dysk docelowy nie ma atrybutu 'LOKALNY'. Z tego powodu Windows prawdopodobnie zawiesi się podczas startu, ponieważ Microsoft nie zaprojektował go do działania z dyskami, które zamiast tego mają atrybut 'WYMIENNY'.\n\nCzy nadal chcesz kontynuować?\n\nInformacja: Atrybut 'LOKALNY/WYMIENNY' to właściwość sprzętowa, która może jedynie zostać zmieniona używając narzędzi dostarczanych przez producenta dysku. Jednak te narzędzia PRAWIE NIGDY nie są udostępniane publicznie..." t MSG_099 "Ograniczenia systemu plików" t MSG_100 "Ten obraz ISO zawiera plik większy niż 4 GB, większy niż maksymalny dozwolony rozmiar dla systemu plików FAT lub FAT32." t MSG_101 "Brakujące wsparcie WIM" -t MSG_102 "Twoja platforma nie może wypakować plików z archiwów WIM. Wypakowanie WIM jest wymagane aby utworzyć bootowalny dysk USB EFI z Windows 7 i Windows Vista. Możesz to naprawić instalując aktualną wersję 7-Zip.\nCzy chcesz odwiedzić stronę pobierania 7-zip?" +t MSG_102 "Twoja platforma nie może wypakować plików z archiwów WIM. Wypakowanie WIM jest wymagane aby utworzyć rozruchowy dysk USB EFI z Windows 7 i Windows Vista. Możesz to naprawić instalując aktualną wersję 7-Zip.\nCzy chcesz odwiedzić stronę pobierania 7-zip?" t MSG_103 "Pobrać %s?" -t MSG_104 "%s lub późniejszy wymaga zainstalowania pliku '%s'.\nPonieważ ten plik jest większy niż 100 KB i zawsze jest obecny na obrazach ISO %s, nie jest dołączony do Rufusa.\n\nRufus może pobrać brakujący plik dla ciebie:\n- Wybierz 'Yes', aby połączyć sie z internetem i pobrać plik\n- Wybierz 'No', jeśli chcesz później ręcznie skopiować ten plik na dysk \n\nInformacja: Plik zostanie pobrany do obecnego folderu i jeśli '%s' istnieje, zostanie użyty ponownie automatycznie." -t MSG_105 "Przerwanie operacji może pozostawic urządzenie w stanie NIEZDATNYM DO UŻYTKU.\nJeśli jesteś pewien, że chcesz przerwać operację, naciśnij TAK. W innym razie naciśnij NIE." +t MSG_104 "%s lub późniejszy wymaga zainstalowania pliku '%s'.\nPonieważ ten plik jest większy niż 100 KB i zawsze jest obecny na obrazach ISO %s, nie jest dołączony do Rufusa.\n\nRufus może pobrać brakujący plik dla ciebie:\n- Wybierz 'Yes', aby połączyć się z internetem i pobrać plik\n- Wybierz 'No', jeśli chcesz później ręcznie skopiować ten plik na dysk \n\nInformacja: Plik zostanie pobrany do obecnego folderu i jeśli '%s' istnieje, zostanie użyty ponownie automatycznie." +t MSG_105 "Przerwanie operacji może pozostawić urządzenie w stanie NIEZDATNYM DO UŻYTKU.\nJeśli jesteś pewien, że chcesz przerwać operację, naciśnij TAK. W innym razie naciśnij NIE." t MSG_106 "Proszę wybrać folder" t MSG_107 "Wszystkie pliki" t MSG_108 "Dziennik Rufusa" @@ -10420,12 +10420,12 @@ t MSG_124 "Brak trwałości" t MSG_125 "Ustaw rozmiar stałej partycji dla nośnika live USB. Ustawienie rozmiaru na 0 wyłącza trwałą partycję." t MSG_126 "Wybierz jednostkę rozmiaru partycji." t MSG_127 "Nie pokazuj więcej tej wiadomości" -t MSG_128 "Ważna informacjoa o %s" +t MSG_128 "Ważna informacja o %s" t MSG_129 "Właśnie utworzyłeś nośnik, który używa bootloadera UEFI:NTFS. Pamiętaj, że aby wystartować z tego medium, MUSISZ WYŁĄCZYĆ \"SECURE BOOT\".\nAby dowiedzieć się więcej dlaczego jest to wymagane, kliknij poniżej przycisk \"Więcej informacji\"." t MSG_130 "Wybór obrazu Windows" t MSG_131 "To ISO zawiera wiele obrazów Windows.\nWybierz obraz, którego chcesz użyć dla tej instalacji:" t MSG_132 "Inny program lub proces używa tego dysku. Czy chcesz go sformatować mimo to?" -t MSG_133 "Rufus wykrył, że próbujesz utworzyć nośnik \"Windows To Go\" bazujący na ISO 1809.\n\nZ powodu *BŁĘDU MICROSOFTU*, ten nośnik ulegnie awarii podczas bootowania systemu (Blue Screen of Death), chyba że ręcznie zastąpisz plik \"WppRecorder.sys\" wersją z 1803.\n\nNależy również pamiętać, że powodem, dla którego Rufus nie może automatycznie rozwiązać tego problemu, jest to, że \"WppRecorder.sys” jest plikiem chronionym prawem autorskim firmy Microsoft, więc nie możemy zgodnie z prawem osadzić kopii pliku w aplikacji." +t MSG_133 "Rufus wykrył, że próbujesz utworzyć nośnik \"Windows To Go\" bazujący na ISO 1809.\n\nZ powodu *BŁĘDU MICROSOFTU*, ten nośnik ulegnie awarii podczas rozruchu systemu (Blue Screen of Death), chyba że ręcznie zastąpisz plik \"WppRecorder.sys\" wersją z 1803.\n\nNależy również pamiętać, że powodem, dla którego Rufus nie może automatycznie rozwiązać tego problemu, jest to, że \"WppRecorder.sys” jest plikiem chronionym prawem autorskim firmy Microsoft, więc nie możemy zgodnie z prawem osadzić kopii pliku w aplikacji." t MSG_134 "Ponieważ MBR został wybrany jako schemat partycji, Rufus może utworzyć partycję o rozmiarze do 2 TB na tym nośniku, co spowoduje, że %s miejsca na dysku będzie niedostępne.\n\nJesteś pewien, że chcesz kontynuować?" t MSG_135 "Wersja" t MSG_136 "Wydanie" @@ -10441,7 +10441,7 @@ t MSG_145 "PowerShell 3.0 albo nowszy wymagany do uruchomienia tego skryptu." t MSG_146 "Czy chcesz przejść do trybu online i go pobrać?" t MSG_148 "Uruchamianie skryptu pobierania..." t MSG_149 "Pobierz obraz ISO" -t MSG_150 "Typ komputera, na którym chcesz użyć bootowalnego dysku. Przed stworzeniem dysku musisz sam określić, czy docelowy komputer posiada BIOS, czy UEFI, inaczej może się nie uruchomić." +t MSG_150 "Typ komputera, na którym chcesz użyć dysku rozruchowego. Przed stworzeniem dysku musisz sam określić, czy docelowy komputer posiada BIOS, czy UEFI, inaczej może się nie uruchomić." t MSG_151 "'UEFI-CSM' oznacza, że urządzenie uruchomi się jedynie w trybie emulacji BIOSu (znanej jako 'Legacy Mode') w UEFI, ale nie w natywnym trybie UEFI." t MSG_152 "'bez CSM' oznacza, że urządzenie uruchomi się jedynie w natywnym trybie UEFI, ale nie w trybie emulacji BIOSu (znanej jako 'Legacy Mode')." t MSG_153 "Wzorzec testowy: 0x%02X" @@ -10452,10 +10452,10 @@ t MSG_157 "Ustawia docelowy system plików" t MSG_158 "Minimalny rozmiar jaki blok będzie zajmował w systemie plików" t MSG_159 "Użyj tego pola, aby ustawić etykietę dysku.\nZnaki międzynarodowe są dozwolone." t MSG_160 "Przełącz opcje zaawansowane" -t MSG_161 "Sprawdź urządzenie pod kątem błędnych sektorów używając wzoru testoego" +t MSG_161 "Sprawdź urządzenie pod kątem błędnych sektorów używając wzoru testowego" t MSG_162 "Odznacz to pole, aby użyć \"powolnej\" metody formatowania" t MSG_163 "Metoda, która zostanie użyta do stworzenia partycji" -t MSG_164 "Metoda, która zostanie użyta, aby uczynić dysk bootowalnym" +t MSG_164 "Metoda, która zostanie użyta, aby uczynić dysk rozruchowym" t MSG_165 "Kliknij, aby wybrać lub pobrać obraz..." t MSG_166 "Zaznacz to pole, aby zezwolić na wyświetlanie etykiet międzynarodowych i ustawić ikonę urządzenia (tworzy plik autorun.inf)" t MSG_167 "Zainstaluj bootloader UEFI, który przeprowadzi weryfikację typu MD5SUM na nośniku" @@ -10476,7 +10476,7 @@ t MSG_182 "Wersja aplikacji, z której korzystasz" t MSG_183 "Twój adres IP" t MSG_184 "W celu generowania poufnych statystyk użytkowania tego programu możemy przechowywać zebrane informacje \\b co najwyżej rok\\b0, jednak nie będziemy udostępniać żadnych z indywidualnych danych osobom trzecim." t MSG_185 "Proces aktualizacji:" -t MSG_186 "Rufus nie instaluje lub uruchamia procesów w tle, dlatego aktualizacje są sprawdzane wylącznie przy starcie głównego okna aplikacji.\\line\nDostęp do internetu jest wymagany podczas sprawdzania aktualizacji." +t MSG_186 "Rufus nie instaluje lub uruchamia procesów w tle, dlatego aktualizacje są sprawdzane wyłącznie przy starcie głównego okna aplikacji.\\line\nDostęp do internetu jest wymagany podczas sprawdzania aktualizacji." t MSG_187 "Obraz nieprawidłowy dla wybranej opcji rozruchowej" t MSG_188 "Obecny obraz nie pasuje do wybranej opcji rozruchowej. Proszę użyć innego obrazu lub wybrać inną opcję rozruchową." t MSG_189 "Ten obraz ISO jest niekompatybilny z wybranym systemem plików" @@ -10486,7 +10486,7 @@ t MSG_192 "Postęp odczytu" t MSG_193 "Pobrano %s" t MSG_194 "Nie można pobrać %s" t MSG_195 "Używanie wbudowanej wersji pliku(ów) %s" -t MSG_196 "WAŻNE: TEN DYSK UŻYWA NIESTANDARDOWEGO ROZMIARU SEKTORA!\n\nStandardowe dyski używają 512-bajtowego rozmiaru sektora, lecz ten dysk używa %d-bajtowego. W wielu przypadkach oznacza to, że NIE będziesz mógł bootować z tego dysku.\nRufus może spróbować utworzyć dysk rozruchowy, ale NIE MA GWARANCJI, że to zadziała." +t MSG_196 "WAŻNE: TEN DYSK UŻYWA NIESTANDARDOWEGO ROZMIARU SEKTORA!\n\nStandardowe dyski używają 512-bajtowego rozmiaru sektora, lecz ten dysk używa %d-bajtowego. W wielu przypadkach oznacza to, że NIE będziesz mógł startować z tego dysku.\nRufus może spróbować utworzyć dysk rozruchowy, ale NIE MA GWARANCJI, że to zadziała." t MSG_197 "Wykryto niestandardowy rozmiar sektora" t MSG_198 "'Windows To Go' może zostać zainstalowany jedynie na dysku ze strukturą GPT jeżeli ma on ustawiony atrybut LOKALNY. Obecny dysk nie został wykryty jako LOKALNY." t MSG_199 "Ta funkcja nie jest dostępna na tej platformie." @@ -10521,7 +10521,7 @@ t MSG_228 "Zapisywanie Master Boot Record..." t MSG_229 "Zapisywanie Partition Boot Record..." t MSG_230 "Kopiowanie plików DOS..." t MSG_231 "Kopiowanie plików ISO: %s" -t MSG_232 "Ustawianie bootowania Win7 EFI (%s)..." +t MSG_232 "Ustawianie rozruchu Win7 EFI (%s)..." t MSG_233 "Finalizacja, proszę czekać..." t MSG_234 "Instalowanie Syslinux %s..." t MSG_235 "Błędne Sektory: %s %d/%d - %0.2f%% (%d/%d/%d błędów)" @@ -10564,7 +10564,7 @@ t MSG_271 "Obliczanie sum kontrolnych obrazu: %s" t MSG_272 "Oblicz sumy kontrolne MD5, SHA1 oraz SHA256 dla wybranego obrazu" t MSG_273 "Zmień język aplikacji" t MSG_274 "Wykryto obraz %s" -t MSG_275 "Obraz, który wybrałeś to obraz 'ISOHybrid'. To znacza iż może zostać zapisany w trybie %s (kopia plików) lub trybie %s (obraz dysku).\nRufus zaleca użycie trybu %s, abyś zawsze miał pełny dostęp do dysku po jego zapisaniu.\nJednak jeśli napotkasz problemy podczas bootowania możesz spróbować zapisać ten obraz ponownie w trybie %s.\n\nProszę wybrać tryb którego chcesz użyć do zapisania tego obrazu:" +t MSG_275 "Obraz, który wybrałeś to obraz 'ISOHybrid'. Oznacza to, że może zostać zapisany w trybie %s (kopia plików) lub trybie %s (obraz dysku).\nRufus zaleca użycie trybu %s, abyś zawsze miał pełny dostęp do dysku po jego zapisaniu.\nJednak jeśli napotkasz problemy podczas rozruchu możesz spróbować zapisać ten obraz ponownie w trybie %s.\n\nProszę wybrać tryb którego chcesz użyć do zapisania tego obrazu:" t MSG_276 "Zapisz w trybie %s (zalecane)" t MSG_277 "Zapisz w trybie %s" t MSG_278 "Sprawdzanie kolidujących procesów..." @@ -10599,8 +10599,8 @@ t MSG_306 "Szybkie zerowanie dysku: %s" t MSG_307 "to może chwilę potrwać" t MSG_308 "Detekcja VHD" t MSG_309 "Skompresowane archiwum" -t MSG_310 "Obraz ISO, który został wybrany używa UEFI i jest dość mały, żeby być zapisanym jako partycja systemowa EFI (ESP). Zapisywanie do ESP zamiast to ogólnej partycji zajmującej cały dysk. Może być bardziej odpowiedni dla niektórych typów instalacji.\n\nProszę wybrać tryb którym chesz zapisać ten obraz:" -t MSG_311 "Użyj %s w głownym oknie aplikacji, żeby włączyć." +t MSG_310 "Wybrany plik ISO używa UEFI i jest na tyle mały, że można go zapisać jako partycję systemową EFI (ESP). Zapisanie na ESP, zamiast zapisywania na ogólnej partycji danych zajmującej cały dysk, może być preferowane dla niektórych typów instalacji.\n\nProszę wybrać tryb, w którym chcesz zapisać ten obraz:" +t MSG_311 "Użyj %s w głównym oknie aplikacji, żeby włączyć." t MSG_312 "Dodatkowe hasze (SHA512)" t MSG_313 "Zapisz do VHD" t MSG_314 "Oblicz sumy kontrolne obrazu" @@ -10623,9 +10623,9 @@ t MSG_333 "Stwórz konto lokalne o nazwie użytkownika:" t MSG_334 "Ustaw taką samą lokalizację i wartości jak obecny użytkownik" t MSG_335 "Dezaktywuj automatyczne szyfrowanie BitLocker" t MSG_336 "Stały dziennik (log)" -t MSG_337 "Dodatkowy plik (\"diskcopy.dll\") musi zostać pobrany od Microsoft, żeby zainstalować MS-DOS:\n- Wybierz opcję \"Tak\", aby pobrać pliku z internetu\n- Wybierz opcję \"Nie\", aby anulować operajcę\n\nUwaga: Plik zostanie pobrany do folderu aplikacji i będzie użyty pownownie, gdy będzie dostępny." +t MSG_337 "Dodatkowy plik (\"diskcopy.dll\") musi zostać pobrany od Microsoft, żeby zainstalować MS-DOS:\n- Wybierz opcję \"Tak\", aby pobrać pliku z internetu\n- Wybierz opcję \"Nie\", aby anulować operację\n\nUwaga: Plik zostanie pobrany do folderu aplikacji i będzie użyty ponownie, gdy będzie dostępny." t MSG_338 "Wykryto unieważniony bootloader UEFI" -t MSG_339 "Rufus wykrył, że wybrany plik ISO zawiera bootloader UEFI, który został unieważniony, i który wygenereuje %s. gdy włączony zostanie \"Secure Boot\" w pełni zaktualizowanym systemie UEFI.\n\n- Jeśli uzyskałeś ten obraz ISO z niezaufanego źródła, powinienneś rozważyć możliwość, że może on zawierać złośliwe oprogramowanie dla UEFI i unikać jego uruchamiania.\n- Jeśli uzyskałeś go z zaufanego źródła, powinieneś spróbować znaleźć nowszą wersję, która nie wygeneruje tego ostrzeżenia." +t MSG_339 "Rufus wykrył, że wybrany plik ISO zawiera bootloader UEFI, który został unieważniony, i który wygeneruje %s. gdy włączony zostanie \"Secure Boot\" w pełni zaktualizowanym systemie UEFI.\n\n- Jeśli uzyskałeś ten obraz ISO z niezaufanego źródła, powinieneś rozważyć możliwość, że może on zawierać złośliwe oprogramowanie dla UEFI i unikać jego uruchamiania.\n- Jeśli uzyskałeś go z zaufanego źródła, powinieneś spróbować znaleźć nowszą wersję, która nie wygeneruje tego ostrzeżenia." t MSG_340 "ekran \"Naruszenie Bezpieczeństwa\"" t MSG_341 "\"Ekran odzyskiwania systemu Windows\" (BSOD) z '%s'" t MSG_342 "Zkompresowany obraz VHDX" @@ -10636,22 +10636,22 @@ t MSG_346 "Ogranicz system Windows do trybu S (NIEZGODNY z obejściem konta onli t MSG_347 "Tryb eksperta" t MSG_348 "Wypakowywanie plików z archiwum: %s" t MSG_349 "Użyj Rufus MBR" -t MSG_900 "Rufus to narzędzie, które pomaga formatować i tworzyć bootowalne dyski flash USB, takie jak klucze USB / pendrive, pendrive'y itp." +t MSG_900 "Rufus to narzędzie, które pomaga formatować i tworzyć dyski rozruchowe flash USB, takie jak klucze USB, pendrive-y itp." t MSG_901 "Oficjalna strona: %s" t MSG_902 "Kod źródłowy: %s" t MSG_903 "Zmiany: %s" t MSG_904 "Ta aplikacja jest objęta licencją na warunkach licencji publicznej GNU (GPL) w wersji 3.\nZ obacz https://www.gnu.org/licenses/gpl-3.0.html, aby uzyskać szczegółowe informacje." t MSG_910 "Sformatuj nośnik używając: FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3" -t MSG_911 "Stwórz bootowalny nośnik USB FreeDOS" +t MSG_911 "Stwórz nośnik rozruchowy USB FreeDOS" t MSG_912 "Twórz dyski rozruchowe z obrazów ISO (Windows, Linux itp.)" t MSG_913 "Twórz dyski rozruchowe z obrazów dysków, włączając skompresowane" t MSG_914 "Stwórz dysk rozruchowy BIOS lub UEFI, włączając bootowalny dysk UEFI NTFS" t MSG_915 "Stwórz dysk 'Windows To Go'" t MSG_916 "Twórz dyski instalacyjne systemu Windows 11 dla komputerów, które nie posiadają modułu TPM ani Secure Boot" -t MSG_917 "Stwórz particje persistent Linuxa" +t MSG_917 "Utwórz trwałe partycje Linux" t MSG_918 "Stwórz obraz VHD/DD z wybranego dysku" t MSG_919 "Oblicz sumy kontrolne MD5, SHA-1, SHA-256 i SHA-512 dla wybranego obrazu" -t MSG_920 "Sprawdź dysk pod względem spójności danych lub wykryj \"nieorginalny\" pendrive" +t MSG_920 "Sprawdź dysk pod względem spójności danych lub wykryj \"nieoryginalny\" pendrive" t MSG_921 "Pobierz oficjalny obraz ISO systemu Microsoft Windows" t MSG_922 "Ściągnij obrazy ISO UEFI Shell" diff --git a/src/rufus.rc b/src/rufus.rc index 7d45ba7649b..059c0c017ed 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2192" +CAPTION "Rufus 4.6.2193" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2192,0 - PRODUCTVERSION 4,6,2192,0 + FILEVERSION 4,6,2193,0 + PRODUCTVERSION 4,6,2193,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2192" + VALUE "FileVersion", "4.6.2193" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2192" + VALUE "ProductVersion", "4.6.2193" END END BLOCK "VarFileInfo" From 248a37e3084f519a02141a72a11289c1d22e7f31 Mon Sep 17 00:00:00 2001 From: Pete Batard Date: Sat, 14 Sep 2024 12:21:22 +0100 Subject: [PATCH 12/30] [cmd] fix hogger invocation for PowerShell * Closes #2556. * Also update issue template, dependabot frequency and signing cert references. * Also harmonize segment addressing code in mbr.S. --- .github/dependabot.yml | 2 +- .github/issue_template.md | 1 + _sign.cmd | 2 +- res/appstore/packme.cmd | 4 ++-- res/mbr/mbr.S | 4 ++-- src/rufus.c | 10 +++++----- src/rufus.rc | 10 +++++----- 7 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 123014908be..5ace4600a1f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,4 +3,4 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "daily" + interval: "weekly" diff --git a/.github/issue_template.md b/.github/issue_template.md index 78f390bceb3..2da326fd5cd 100644 --- a/.github/issue_template.md +++ b/.github/issue_template.md @@ -23,6 +23,7 @@ YOU HAVE BEEN WARNED. Checklist --------- +- [ ] I have been made aware that if my problem can be summarized as _"I've created or tried to create a media using Rufus, and ..."_, and I am not including a log, **this issue will be summarily closed**. - [ ] I looked at https://github.com/pbatard/rufus/wiki/FAQ to see if my question has already been answered. - [ ] I performed a search in the issue tracker for similar issues using keywords relevant to my problem, such as the error message I got from the log. - [ ] I clicked the 'Log' button (🗒️) or pressed Ctrl-L in Rufus, or used [DebugView](https://learn.microsoft.com/en-us/sysinternals/downloads/debugview), and copy/pasted the log into the section that says `` below. diff --git a/_sign.cmd b/_sign.cmd index ca68f4f1aa7..1da906b6545 100644 --- a/_sign.cmd +++ b/_sign.cmd @@ -1,2 +1,2 @@ @echo off -"C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64\signtool" sign /v /sha1 3dbc3a2a0e9ce8803b422cfdbc60acd33164965d /fd SHA256 /tr http://sha256timestamp.ws.symantec.com/sha256/timestamp /td SHA256 %* +"C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64\signtool" sign /v /sha1 fc4686753937a93fdcd48c2bb4375e239af92dcb /fd SHA256 /tr http://timestamp.acs.microsoft.com /td SHA256 %* diff --git a/res/appstore/packme.cmd b/res/appstore/packme.cmd index 04b8f0dc956..193124aaaef 100644 --- a/res/appstore/packme.cmd +++ b/res/appstore/packme.cmd @@ -27,7 +27,7 @@ del /q *.map >NUL 2>&1 set WDK_PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64 set ZIP_PATH=C:\Program Files\7-Zip -set SIGNATURE_SHA1=3dbc3a2a0e9ce8803b422cfdbc60acd33164965d +set SIGNATURE_SHA1=fc4686753937a93fdcd48c2bb4375e239af92dcb set MANIFEST=AppxManifest.xml set ARCHS=x86 x64 arm arm64 set DEFAULT_SCALE=200 @@ -98,7 +98,7 @@ if "%VERSION_OVERRIDE%"=="" ( echo Will create %VERSION% AppStore Bundle pause -"%WDK_PATH%\signtool" sign /v /sha1 %SIGNATURE_SHA1% /fd SHA256 /tr http://sha256timestamp.ws.symantec.com/sha256/timestamp /td SHA256 *.exe +"%WDK_PATH%\signtool" sign /v /sha1 %SIGNATURE_SHA1% /fd SHA256 /tr http://timestamp.acs.microsoft.com /td SHA256 *.exe if ERRORLEVEL 1 goto out echo [Files]> bundle.map diff --git a/res/mbr/mbr.S b/res/mbr/mbr.S index 746eb2b29bf..312fa471850 100644 --- a/res/mbr/mbr.S +++ b/res/mbr/mbr.S @@ -67,9 +67,9 @@ mbr: push es mov ds, ax # Original MBR is in segment 0 mov bx, 0x0413 # McAfee thinks we are a virus if we use 413 directly... - mov ax,[bx] + mov ax,ds:[bx] dec ax # Remove 1KB from the RAM for our copy - mov [bx],ax + mov ds:[bx],ax shl ax, 6 # Convert to segment address sub ax, MBR_ADDR>>4 # For convenience of disassembly, so that irrespective mov es, ax # of the segment, our base address remains MBR_ADDR diff --git a/src/rufus.c b/src/rufus.c index 37747e2548d..7c017afcd3d 100755 --- a/src/rufus.c +++ b/src/rufus.c @@ -65,7 +65,7 @@ enum bootcheck_return { BOOTCHECK_GENERAL_ERROR = -3, }; -static const char* cmdline_hogger = "rufus.com"; +static const char* cmdline_hogger = ".\\rufus.com"; static const char* ep_reg = "Software\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer"; static const char* vs_reg = "Software\\Microsoft\\VisualStudio"; static const char* arch_name[ARCH_MAX] = { @@ -3125,13 +3125,13 @@ static HANDLE SetHogger(void) int i; hog_data = GetResource(hMainInstance, MAKEINTRESOURCEA(IDR_XT_HOGGER), - _RT_RCDATA, cmdline_hogger, &hog_size, FALSE); + _RT_RCDATA, &cmdline_hogger[2], &hog_size, FALSE); if (hog_data != NULL) { // Create our synchronisation mutex hogmutex = CreateMutexA(NULL, TRUE, "Global/Rufus_CmdLine"); // Extract the hogger resource - hFile = CreateFileA(cmdline_hogger, GENERIC_READ|GENERIC_WRITE, FILE_SHARE_READ, + hFile = CreateFileA(&cmdline_hogger[2], GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); if (hFile != INVALID_HANDLE_VALUE) { // coverity[check_return] @@ -3140,7 +3140,7 @@ static HANDLE SetHogger(void) safe_closehandle(hFile); // Now launch the file from the commandline, by simulating keypresses - input = (INPUT*)calloc(strlen(cmdline_hogger)+1, sizeof(INPUT)); + input = (INPUT*)calloc(strlen(cmdline_hogger) + 1, sizeof(INPUT)); if (input != NULL) { for (i = 0; i < (int)strlen(cmdline_hogger); i++) { input[i].type = INPUT_KEYBOARD; @@ -4083,7 +4083,7 @@ extern int TestHashes(void); ReleaseMutex(hogmutex); safe_closehandle(hogmutex); // Unconditional delete with retry, just in case... - for (i = 0; (!DeleteFileA(cmdline_hogger)) && (i <= 10); i++) + for (i = 0; (!DeleteFileA(&cmdline_hogger[2])) && (i <= 10); i++) Sleep(200); } // Kill the update check thread if running diff --git a/src/rufus.rc b/src/rufus.rc index 059c0c017ed..e81d2d42c6b 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2193" +CAPTION "Rufus 4.6.2194" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2193,0 - PRODUCTVERSION 4,6,2193,0 + FILEVERSION 4,6,2194,0 + PRODUCTVERSION 4,6,2194,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2193" + VALUE "FileVersion", "4.6.2194" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2193" + VALUE "ProductVersion", "4.6.2194" END END BLOCK "VarFileInfo" From f453dc272b93adb0d63dd045197f4661fc653211 Mon Sep 17 00:00:00 2001 From: Pete Batard Date: Mon, 30 Sep 2024 17:38:47 +0100 Subject: [PATCH 13/30] [misc] fix a potential double free and avoid nonsensical error messages * buf could be freed twice in iso.c. * Using HRESULT_CODE(error_code) in WindowsErrorString() could lead to "Error: SUCCESS" messages. * Closes #2524. * Also try to address potential issues that appear to have been seen in the wild. --- src/dev.c | 4 ++-- src/drive.c | 6 ++++-- src/hash.c | 7 +++++-- src/iso.c | 2 +- src/rufus.rc | 10 +++++----- src/stdio.c | 6 +++--- 6 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/dev.c b/src/dev.c index 801fb12430e..5ea2b2a06ab 100644 --- a/src/dev.c +++ b/src/dev.c @@ -877,8 +877,8 @@ BOOL GetDevices(DWORD devnum) continue; } - hDrive = CreateFileA(devint_detail_data->DevicePath, GENERIC_READ|GENERIC_WRITE, - FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + hDrive = CreateFileWithTimeout(devint_detail_data->DevicePath, GENERIC_READ|GENERIC_WRITE, + FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL, 3000); if(hDrive == INVALID_HANDLE_VALUE) { uprintf("Could not open '%s': %s", devint_detail_data->DevicePath, WindowsErrorString()); continue; diff --git a/src/drive.c b/src/drive.c index 8d0514d8fde..5f378e2424a 100644 --- a/src/drive.c +++ b/src/drive.c @@ -1894,10 +1894,12 @@ BOOL GetDrivePartitionData(DWORD DriveIndex, char* FileSystemName, DWORD FileSys SelectedDrive.SectorsPerTrack = DiskGeometry->Geometry.SectorsPerTrack; SelectedDrive.MediaType = DiskGeometry->Geometry.MediaType; - suprintf("Disk type: %s, Disk size: %s, Sector size: %d bytes", (SelectedDrive.MediaType == FixedMedia)?"FIXED":"Removable", + suprintf("Disk type: %s, Disk size: %s, Sector size: %d bytes", + (SelectedDrive.MediaType == FixedMedia) ? "FIXED" : "Removable", SizeToHumanReadable(SelectedDrive.DiskSize, FALSE, TRUE), SelectedDrive.SectorSize); suprintf("Cylinders: %" PRIi64 ", Tracks per cylinder: %d, Sectors per track: %d", DiskGeometry->Geometry.Cylinders, DiskGeometry->Geometry.TracksPerCylinder, DiskGeometry->Geometry.SectorsPerTrack); + assert(SelectedDrive.SectorSize != 0); r = DeviceIoControl(hPhysical, IOCTL_DISK_GET_DRIVE_LAYOUT_EX, NULL, 0, layout, sizeof(layout), &size, NULL ); if (!r || size <= 0) { @@ -1965,7 +1967,7 @@ BOOL GetDrivePartitionData(DWORD DriveIndex, char* FileSystemName, DWORD FileSys SizeToHumanReadable(DriveLayout->PartitionEntry[i].PartitionLength.QuadPart, TRUE, FALSE), DriveLayout->PartitionEntry[i].PartitionLength.QuadPart, DriveLayout->PartitionEntry[i].StartingOffset.QuadPart / SelectedDrive.SectorSize, - DriveLayout->PartitionEntry[i].Mbr.BootIndicator?"Yes":"No"); + DriveLayout->PartitionEntry[i].Mbr.BootIndicator ? "Yes" : "No"); // suprintf(" GUID: %s", GuidToString(&DriveLayout->PartitionEntry[i].Mbr.PartitionId)); SelectedDrive.FirstDataSector = min(SelectedDrive.FirstDataSector, (DWORD)(DriveLayout->PartitionEntry[i].StartingOffset.QuadPart / SelectedDrive.SectorSize)); diff --git a/src/hash.c b/src/hash.c index a99d297bfa8..76315edc54d 100644 --- a/src/hash.c +++ b/src/hash.c @@ -1607,8 +1607,7 @@ static int cmp_pe_section(const void* arg1, const void* arg2) * @len: Size of @efi * @regp: Pointer to a list of regions * - * Parse image binary in PE32(+) format, assuming that sanity of PE image - * has been checked by a caller. + * Parse image binary in PE32(+) format. * * Return: TRUE on success, FALSE on error */ @@ -1623,7 +1622,11 @@ BOOL efi_image_parse(uint8_t* efi, size_t len, struct efi_image_regions** regp) uint32_t align, size, authsz; size_t bytes_hashed; + if (len < 0x80) + return FALSE; dos = (void*)efi; + if (dos->e_lfanew > len - 0x40) + return FALSE; nt = (void*)(efi + dos->e_lfanew); authsz = 0; diff --git a/src/iso.c b/src/iso.c index a5b978be362..485de74cc38 100644 --- a/src/iso.c +++ b/src/iso.c @@ -1268,7 +1268,7 @@ BOOL ExtractISO(const char* src_iso, const char* dest_dir, BOOL scan) sl_index = i; } } - free(buf); + safe_free(buf); } DeleteFileU(isolinux_tmp); } diff --git a/src/rufus.rc b/src/rufus.rc index e81d2d42c6b..9a2845eb1c0 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2194" +CAPTION "Rufus 4.6.2195" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2194,0 - PRODUCTVERSION 4,6,2194,0 + FILEVERSION 4,6,2195,0 + PRODUCTVERSION 4,6,2195,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2194" + VALUE "FileVersion", "4.6.2195" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2194" + VALUE "ProductVersion", "4.6.2195" END END BLOCK "VarFileInfo" diff --git a/src/stdio.c b/src/stdio.c index 59909196c37..4004575189f 100644 --- a/src/stdio.c +++ b/src/stdio.c @@ -260,8 +260,8 @@ const char *WindowsErrorString(void) // coverity[var_deref_model] size = FormatMessageU(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS | ((hModule != NULL) ? FORMAT_MESSAGE_FROM_HMODULE : 0), hModule, - HRESULT_CODE(error_code), MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US), - &err_string[presize], (DWORD)(sizeof(err_string)-strlen(err_string)), NULL); + error_code, MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US), + &err_string[presize], (DWORD)(sizeof(err_string) - strlen(err_string)), NULL); if (size == 0) { format_error = GetLastError(); switch (format_error) { @@ -519,7 +519,7 @@ HANDLE CreateFileWithTimeout(LPCSTR lpFileName, DWORD dwDesiredAccess, DWORD dwS if (hThread != NULL) { if (WaitForSingleObject(hThread, dwTimeOut) == WAIT_TIMEOUT) { CancelSynchronousIo(hThread); - WaitForSingleObject(hThread, INFINITE); + WaitForSingleObject(hThread, 30000); params.dwError = WAIT_TIMEOUT; } CloseHandle(hThread); From c5d61f6696506e5397ea67f8d31b4040f374c57e Mon Sep 17 00:00:00 2001 From: Pete Batard Date: Thu, 3 Oct 2024 20:25:19 +0100 Subject: [PATCH 14/30] [iso] add SBAT revocation validation and reporting --- src/hash.c | 123 +++++++++++++++++++++++++++++++-------------------- src/iso.c | 47 +++++++++++--------- src/parser.c | 51 +++++++++++++++++++++ src/rufus.c | 58 +++++++++++++----------- src/rufus.h | 16 ++++--- src/rufus.rc | 10 ++--- src/stdlg.c | 16 ++++++- 7 files changed, 217 insertions(+), 104 deletions(-) diff --git a/src/hash.c b/src/hash.c index 76315edc54d..84ab9c5c6ea 100644 --- a/src/hash.c +++ b/src/hash.c @@ -115,7 +115,7 @@ uint64_t md5sum_totalbytes; StrArray modified_files = { 0 }; extern int default_thread_priority; -extern const char* efi_bootname[ARCH_MAX]; +extern const char* efi_archname[ARCH_MAX]; /* * Rotate 32 or 64 bit integers by n bytes. @@ -1760,36 +1760,18 @@ BOOL efi_image_parse(uint8_t* efi, size_t len, struct efi_image_regions** regp) * for some part of this but you'd be very, very wrong since the PE sections it * feeds to the hash function does include the PE header checksum field... */ -BOOL PE256File(const char* path, uint8_t* hash) +BOOL PE256Buffer(uint8_t* buf, uint32_t len, uint8_t* hash) { BOOL r = FALSE; HASH_CONTEXT hash_ctx = { {0} }; int i; - uint32_t rb; - uint8_t* buf = NULL; - struct __stat64 stat64 = { 0 }; struct efi_image_regions* regs = NULL; - if ((path == NULL) || (hash == NULL)) - goto out; - - /* Filter anything that would be out of place as a EFI bootloader */ - if (_stat64U(path, &stat64) != 0) { - uprintf("Could not open '%s", path); - goto out; - } - if ((stat64.st_size < 1 * KB) || (stat64.st_size > 64 * MB)) { - uprintf("'%s' is either too small or too large for PE-256", path); - goto out; - } - - /* Read the executable into a memory buffer */ - rb = read_file(path, &buf); - if (rb < 1 * KB) + if ((buf == NULL) || (len == 0) || (len < 1 * KB) || (len > 64 * MB) || (hash == NULL)) goto out; /* Isolate the PE sections to hash */ - if (!efi_image_parse(buf, rb, ®s)) + if (!efi_image_parse(buf, len, ®s)) goto out; /* Hash the relevant PE data */ @@ -1803,7 +1785,6 @@ BOOL PE256File(const char* path, uint8_t* hash) out: free(regs); - free(buf); return r; } @@ -2098,36 +2079,82 @@ BOOL IsFileInDB(const char* path) return FALSE; } -int IsBootloaderRevoked(const char* path) +BOOL IsRevokedBySbat(uint8_t* buf, uint32_t len) +{ + static const char section_name[IMAGE_SIZEOF_SHORT_NAME] = { '.', 's', 'b', 'a', 't', '\0', '\0', '\0' }; + char* sbat = NULL, * version_str; + uint32_t i, j; + sbat_entry_t entry; + IMAGE_SECTION_HEADER* section_header; + + if (buf == NULL || len < 0x100 || sbat_entries == NULL) + return FALSE; + + for (i = 0x40; i < MIN(len, 0x800); i++) { + if (memcmp(section_name, &buf[i], sizeof(section_name)) == 0) { + section_header = (IMAGE_SECTION_HEADER*)&buf[i]; + if (section_header->SizeOfRawData >= len || section_header->PointerToRawData >= len) + return TRUE; + sbat = (char*)&buf[section_header->PointerToRawData]; + break; + } + } + if (sbat == NULL) + return FALSE; + + for (i = 0; sbat[i] != '\0'; ) { + while (sbat[i] == '\n') + i++; + if (sbat[i] == '\0') + break; + entry.product = &sbat[i]; + for (; sbat[i] != ',' && sbat[i] != '\0' && sbat[i] != '\n'; i++); + if (sbat[i] == '\0' || sbat[i] == '\n') + break; + sbat[i++] = '\0'; + version_str = &sbat[i]; + for (; sbat[i] != ',' && sbat[i] != '\0' && sbat[i] != '\n'; i++); + sbat[i++] = '\0'; + entry.version = atoi(version_str); + for (; sbat[i] != '\0' && sbat[i] != '\n'; i++); + if (entry.version == 0) + continue; + for (j = 0; sbat_entries[j].product != NULL; j++) { + if (strcmp(entry.product, sbat_entries[j].product) == 0 && entry.version < sbat_entries[j].version) + return TRUE; + } + } + + return FALSE; +} + +int IsBootloaderRevoked(uint8_t* buf, uint32_t len) { - version_t* ver; uint32_t i; uint8_t hash[SHA256_HASHSIZE]; - if (!PE256File(path, hash)) + if (!PE256Buffer(buf, len, hash)) return -1; + // Check for UEFI DBX revocation for (i = 0; i < ARRAYSIZE(pe256dbx); i += SHA256_HASHSIZE) if (memcmp(hash, &pe256dbx[i], SHA256_HASHSIZE) == 0) return 1; + // Check for Microsoft SSP revocation for (i = 0; i < pe256ssp_size * SHA256_HASHSIZE; i += SHA256_HASHSIZE) if (memcmp(hash, &pe256ssp[i], SHA256_HASHSIZE) == 0) return 2; - ver = GetExecutableVersion(path); - // Blanket filter for Windows 10 1607 (excluded) to Windows 10 20H1 (excluded) - // TODO: Revoke all bootloaders prior to 2023.05 once Microsoft does -// uprintf("Found UEFI bootloader version: %d.%d.%d.%d", ver->Major, ver->Minor, ver->Micro, ver->Nano); - if (ver != NULL && ver->Major == 10 && ver->Minor == 0 && ver->Micro > 14393 && ver->Micro < 19041) + // Check for SBAT revocation + if (IsRevokedBySbat(buf, len)) return 3; + return 0; } void PrintRevokedBootloaderInfo(void) { - uprintf("Found %d officially revoked UEFI bootloaders from embedded list", sizeof(pe256dbx) / SHA256_HASHSIZE); - if (ParseSKUSiPolicy()) + uprintf("Found %d revoked UEFI bootloaders from embedded list", sizeof(pe256dbx) / SHA256_HASHSIZE); + if (ParseSKUSiPolicy() && pe256ssp_size != 0) uprintf("Found %d additional revoked UEFI bootloaders from this system's SKUSiPolicy.p7b", pe256ssp_size); - else - uprintf("WARNING: Could not parse this system's SkuSiPolicy.p7b for additional revoked UEFI bootloaders"); } /* @@ -2148,8 +2175,8 @@ void UpdateMD5Sum(const char* dest_dir, const char* md5sum_name) intptr_t pos; uint32_t i, j, size, md5_size, new_size; uint8_t sum[MD5_HASHSIZE]; - char md5_path[64], path1[64], path2[64], *md5_data = NULL, *new_data = NULL, *str_pos; - char *d, *s, *p; + char md5_path[64], path1[64], path2[64], bootloader_name[32]; + char *md5_data = NULL, *new_data = NULL, *str_pos, *d, *s, *p; if (!img_report.has_md5sum && !validate_md5sum) goto out; @@ -2200,12 +2227,13 @@ void UpdateMD5Sum(const char* dest_dir, const char* md5sum_name) } s = md5_data; // Extract the MD5Sum bootloader(s) - for (i = 1; i < ARRAYSIZE(efi_bootname); i++) { - static_sprintf(path1, "%s\\efi\\boot\\%s", dest_dir, efi_bootname[i]); + for (i = 1; i < ARRAYSIZE(efi_archname); i++) { + static_sprintf(bootloader_name, "boot%s.efi", efi_archname[i]); + static_sprintf(path1, "%s\\efi\\boot\\boot%s.efi", dest_dir, efi_archname[i]); if (!PathFileExistsA(path1)) continue; res_data = (BYTE*)GetResource(hMainInstance, MAKEINTRESOURCEA(IDR_MD5_BOOT + i), - _RT_RCDATA, efi_bootname[i], &res_size, FALSE); + _RT_RCDATA, bootloader_name, &res_size, FALSE); static_strcpy(path2, path1); path2[strlen(path2) - 4] = 0; static_strcat(path2, "_original.efi"); @@ -2232,20 +2260,21 @@ void UpdateMD5Sum(const char* dest_dir, const char* md5sum_name) } // Rename the original bootloaders if present in md5sum.txt for (p = md5_data; (p = StrStrIA(p, " ./efi/boot/boot")) != NULL; ) { - for (i = 1; i < ARRAYSIZE(efi_bootname); i++) { - if (p[12 + strlen(efi_bootname[i])] != 0x0a) + for (i = 1; i < ARRAYSIZE(efi_archname); i++) { + static_sprintf(bootloader_name, "boot%s.efi", efi_archname[i]); + if (p[12 + strlen(bootloader_name)] != 0x0a) continue; - p[12 + strlen(efi_bootname[i])] = 0; - if (lstrcmpiA(&p[12], efi_bootname[i]) == 0) { - size = (uint32_t)(p - s) + 12 + (uint32_t)strlen(efi_bootname[i]) - 4; + p[12 + strlen(bootloader_name)] = 0; + if (lstrcmpiA(&p[12], bootloader_name) == 0) { + size = (uint32_t)(p - s) + 12 + (uint32_t)strlen(bootloader_name) - 4; memcpy(d, s, size); d = &d[size]; strcpy(d, "_original.efi\n"); new_size += 9; d = &d[14]; - s = &p[12 + strlen(efi_bootname[i]) + 1]; + s = &p[12 + strlen(bootloader_name) + 1]; } - p[12 + strlen(efi_bootname[i])] = 0x0a; + p[12 + strlen(bootloader_name)] = 0x0a; } p = &p[12]; } diff --git a/src/iso.c b/src/iso.c index 485de74cc38..eed8c5a3d18 100644 --- a/src/iso.c +++ b/src/iso.c @@ -98,9 +98,8 @@ const char* md5sum_name[2] = { "md5sum.txt", "MD5SUMS" }; static const char* casper_dirname = "/casper"; static const char* proxmox_dirname = "/proxmox"; const char* efi_dirname = "/efi/boot"; -const char* efi_bootname[ARCH_MAX] = { - "boot.efi", "bootia32.efi", "bootx64.efi", "bootarm.efi", "bootaa64.efi", "bootia64.efi", - "bootriscv32.efi", "bootriscv64.efi", "bootriscv128.efi", "bootebc.efi" }; +const char* efi_bootname[3] = { "boot", "grub", "mm" }; +const char* efi_archname[ARCH_MAX] = { "", "ia32", "x64", "arm", "aa64", "ia64", "riscv64", "ebc" }; static const char* sources_str = "/sources"; static const char* wininst_name[] = { "install.wim", "install.esd", "install.swm" }; // We only support GRUB/BIOS (x86) that uses a standard config dir (/boot/grub/i386-pc/) @@ -146,8 +145,8 @@ static __inline char* sanitize_filename(char* filename, BOOL* is_identical) } // Must start after the drive part (D:\...) so that we don't eliminate the first column - for (i=2; i 12) + if (strlen(bootloader_name) > 12) continue; - for (j = 0, k = 0; efi_bootname[i][j] != 0; j++) { - if (efi_bootname[i][j] == '.') { + for (j = 0, k = 0; bootloader_name[j] != 0; j++) { + if (bootloader_name[j] == '.') { while (k < 8) name[k++] = ' '; } else { - name[k++] = toupper(efi_bootname[i][j]); + name[k++] = toupper(bootloader_name[j]); } } c = libfat_searchdir(lf_fs, dc, name, &direntry); if (c > 0) { if (!ret) uprintf(" Detected EFI bootloader(s) (from '%s'):", img_report.efi_img_path); - uprintf(" ● '%s'", efi_bootname[i]); + uprintf(" ● '%s'", bootloader_name); ret = TRUE; } } diff --git a/src/parser.c b/src/parser.c index c331167ee47..2cb09ccf013 100644 --- a/src/parser.c +++ b/src/parser.c @@ -1544,3 +1544,54 @@ int sanitize_label(char* label) return 0; } + +/* + * Parse an sbat_level.txt file and returns an array of (product_name, min_version) tuples. + * Array must be freed by caller. + */ +sbat_entry_t* GetSbatEntries(char* sbatlevel) +{ + BOOL eol, eof; + char* version_str; + uint32_t i, num_entries; + sbat_entry_t* sbat_entries; + + if (sbatlevel == NULL) + return FALSE; + + num_entries = 0; + for (i = 0; sbatlevel[i] != '\0'; i++) + if (sbatlevel[i] == '\n') + num_entries++; + + sbat_entries = calloc(num_entries + 2, sizeof(sbat_entry_t)); + if (sbat_entries == NULL) + return NULL; + + num_entries = 0; + for (i = 0; sbatlevel[i] != '\0'; ) { + while (sbatlevel[i] == '\n') + i++; + if (sbatlevel[i] == '\0') + break; + sbat_entries[num_entries].product = &sbatlevel[i]; + for (; sbatlevel[i] != ',' && sbatlevel[i] != '\0' && sbatlevel[i] != '\n'; i++); + if (sbatlevel[i] == '\0' || sbatlevel[i] == '\n') + break; + sbatlevel[i++] = '\0'; + version_str = &sbatlevel[i]; + for (; sbatlevel[i] != ',' && sbatlevel[i] != '\0' && sbatlevel[i] != '\n'; i++); + eol = (sbatlevel[i] == '\0' || sbatlevel[i] == '\n'); + eof = (sbatlevel[i] == '\0'); + sbatlevel[i] = '\0'; + if (!eof) + i++; + sbat_entries[num_entries].version = atoi(version_str); + if (!eol) + for (; sbatlevel[i] != '\0' && sbatlevel[i] != '\n'; i++); + if (sbat_entries[num_entries].version != 0) + num_entries++; + } + + return sbat_entries; +} diff --git a/src/rufus.c b/src/rufus.c index 7c017afcd3d..bb4647c01d4 100755 --- a/src/rufus.c +++ b/src/rufus.c @@ -69,7 +69,7 @@ static const char* cmdline_hogger = ".\\rufus.com"; static const char* ep_reg = "Software\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer"; static const char* vs_reg = "Software\\Microsoft\\VisualStudio"; static const char* arch_name[ARCH_MAX] = { - "unknown", "x86_32", "x86_64", "ARM", "ARM64", "Itanic", "RISC-V 32", "RISC-V 64", "RISC-V 128", "EBC" }; + "unknown", "x86_32", "x86_64", "ARM", "ARM64", "Itanic", "RISC-V 64", "EBC" }; static BOOL existing_key = FALSE; // For LGP set/restore static BOOL size_check = TRUE; static BOOL log_displayed = FALSE; @@ -141,11 +141,13 @@ char embedded_sl_version_ext[2][32]; char ClusterSizeLabel[MAX_CLUSTER_SIZES][64]; char msgbox[1024], msgbox_title[32], *ini_file = NULL, *image_path = NULL, *short_image_path; char *archive_path = NULL, image_option_txt[128], *fido_url = NULL, *save_image_type = NULL; +char* sbat_level_txt = NULL; StrArray BlockingProcessList, ImageList; // Number of steps for each FS for FCC_STRUCTURE_PROGRESS const int nb_steps[FS_MAX] = { 5, 5, 12, 1, 10, 1, 1, 1, 1 }; const char* flash_type[BADLOCKS_PATTERN_TYPES] = { "SLC", "MLC", "TLC" }; RUFUS_DRIVE rufus_drive[MAX_DRIVES] = { 0 }; +sbat_entry_t* sbat_entries = NULL; // TODO: Remember to update copyright year in stdlg's AboutCallback() WM_INITDIALOG, // localization_data.sh and the .rc when the year changes! @@ -1210,15 +1212,9 @@ static uint8_t FindArch(const char* path) case IMAGE_FILE_MACHINE_IA64: ret = ARCH_IA_64; break; - case IMAGE_FILE_MACHINE_RISCV32: - ret = ARCH_RISCV_32; - break; case IMAGE_FILE_MACHINE_RISCV64: ret = ARCH_RISCV_64; break; - case IMAGE_FILE_MACHINE_RISCV128: - ret = ARCH_RISCV_128; - break; case IMAGE_FILE_MACHINE_EBC: ret = ARCH_EBC; break; @@ -1298,7 +1294,7 @@ DWORD WINAPI ImageScanThread(LPVOID param) arch = FindArch(tmp_path); if (arch != 0) { uprintf(" Image contains a%s %s EFI boot manager", - (arch >= ARCH_RISCV_32 && arch <= ARCH_RISCV_128) ? "" : "n", arch_name[arch]); + (arch == ARCH_RISCV_64) ? "" : "n", arch_name[arch]); img_report.has_efi = 1 | (1 << arch); img_report.has_bootmgr_efi = TRUE; img_report.wininst_index = 1; @@ -1607,26 +1603,36 @@ static DWORD WINAPI BootCheckThread(LPVOID param) // Check UEFI bootloaders for revocation if (IS_EFI_BOOTABLE(img_report)) { - // coverity[swapped_arguments] - if (GetTempFileNameU(temp_dir, APPLICATION_NAME, 0, tmp) != 0) { - // ExtractISOFile() is case sensitive, so we must use - for (i = 0; i < ARRAYSIZE(img_report.efi_boot_path) && img_report.efi_boot_path[i][0] != 0; i++) { - if (ExtractISOFile(image_path, img_report.efi_boot_path[i], tmp, FILE_ATTRIBUTE_NORMAL) == 0) { - uprintf("Warning: Failed to extract '%s' to check for UEFI revocation", img_report.efi_boot_path[i]); - continue; - } - r = IsBootloaderRevoked(tmp); - if (r > 0) { - is_bootloader_revoked = TRUE; - r = MessageBoxExU(hMainDialog, lmprintf(MSG_339, - (r == 1) ? lmprintf(MSG_340) : lmprintf(MSG_341, "Error code: 0xc0000428")), - lmprintf(MSG_338), MB_OKCANCEL | MB_ICONWARNING | MB_IS_RTL, selected_langid); - if (r == IDCANCEL) - goto out; + uint8_t* buf = NULL; + uint32_t len; + const char* msg; + + for (i = 0; i < ARRAYSIZE(img_report.efi_boot_path) && img_report.efi_boot_path[i][0] != 0; i++) { + len = ReadISOFileToBuffer(image_path, img_report.efi_boot_path[i], &buf); + if (len == 0) { + uprintf("Warning: Failed to extract '%s' to check for UEFI revocation", img_report.efi_boot_path[i]); + continue; + } + r = IsBootloaderRevoked(buf, len); + safe_free(buf); + if (r > 0) { + uprintf("Warning: '%s' is revoked by %s", img_report.efi_boot_path[i], + (r == 1) ? "UEFI DBX" : ((r == 2) ? "Windows SSP" : "Linux SBAT")); + is_bootloader_revoked = TRUE; + switch (r) { + case 2: + msg = lmprintf(MSG_341, "Error code: 0xc0000428"); + break; + default: + msg = lmprintf(MSG_340); break; } + r = MessageBoxExU(hMainDialog, lmprintf(MSG_339, msg), lmprintf(MSG_338), + MB_OKCANCEL | MB_ICONWARNING | MB_IS_RTL, selected_langid); + if (r == IDCANCEL) + goto out; + break; } - DeleteFileU(tmp); } } @@ -4105,6 +4111,8 @@ extern int TestHashes(void); safe_free(fido_url); safe_free(fido_script); safe_free(pe256ssp); + safe_free(sbat_entries); + safe_free(sbat_level_txt); if (argv != NULL) { for (i = 0; i < argc; i++) safe_free(argv[i]); diff --git a/src/rufus.h b/src/rufus.h index 471de019667..3fafca6601a 100644 --- a/src/rufus.h +++ b/src/rufus.h @@ -378,9 +378,7 @@ enum ArchType { ARCH_ARM_32, ARCH_ARM_64, ARCH_IA_64, - ARCH_RISCV_32, ARCH_RISCV_64, - ARCH_RISCV_128, ARCH_EBC, ARCH_MAX }; @@ -391,7 +389,7 @@ typedef struct { char cfg_path[128]; // path to the ISO's isolinux.cfg char reactos_path[128]; // path to the ISO's freeldr.sys or setupldr.sys char wininst_path[MAX_WININST][64]; // path to the Windows install image(s) - char efi_boot_path[ARCH_MAX][30]; // paths of detected UEFI bootloaders + char efi_boot_path[64][32]; // paths of detected UEFI bootloaders char efi_img_path[128]; // path to an efi.img file uint64_t image_size; uint64_t projected_size; @@ -487,6 +485,12 @@ typedef struct { uint32_t* address; // 32-bit will do, as we're not dealing with >4GB DLLs... } dll_resolver_t; +/* SBAT entry */ +typedef struct { + char* product; + uint32_t version; +} sbat_entry_t; + /* Alignment macro */ #if defined(__GNUC__) #define ALIGNED(m) __attribute__ ((__aligned__(m))) @@ -687,6 +691,7 @@ extern size_t ubuffer_pos; extern const int nb_steps[FS_MAX]; extern float fScale; extern windows_version_t WindowsVersion; +extern sbat_entry_t* sbat_entries; extern int dialog_showing, force_update, fs_type, boot_type, partition_type, target_type; extern unsigned long syslinux_ldlinux_len[2]; extern char ubuffer[UBUFFER_SIZE], embedded_sl_version_str[2][12]; @@ -798,11 +803,11 @@ extern HANDLE CreateFileWithTimeout(LPCSTR lpFileName, DWORD dwDesiredAccess, DW HANDLE hTemplateFile, DWORD dwTimeOut); extern BOOL SetThreadAffinity(DWORD_PTR* thread_affinity, size_t num_threads); extern BOOL HashFile(const unsigned type, const char* path, uint8_t* sum); -extern BOOL PE256File(const char* path, uint8_t* hash); +extern BOOL PE256Buffer(uint8_t* buf, uint32_t len, uint8_t* hash); extern void UpdateMD5Sum(const char* dest_dir, const char* md5sum_name); extern BOOL HashBuffer(const unsigned type, const uint8_t* buf, const size_t len, uint8_t* sum); extern BOOL IsFileInDB(const char* path); -extern int IsBootloaderRevoked(const char* path); +extern int IsBootloaderRevoked(uint8_t* buf, uint32_t len); extern void PrintRevokedBootloaderInfo(void); extern BOOL IsBufferInDB(const unsigned char* buf, const size_t len); #define printbits(x) _printbits(sizeof(x), &x, 0) @@ -825,6 +830,7 @@ extern HANDLE CreatePreallocatedFile(const char* lpFileName, DWORD dwDesiredAcce DWORD dwShareMode, LPSECURITY_ATTRIBUTES lpSecurityAttributes, DWORD dwCreationDisposition, DWORD dwFlagsAndAttributes, LONGLONG fileSize); extern uint32_t ResolveDllAddress(dll_resolver_t* resolver); +extern sbat_entry_t* GetSbatEntries(char* sbatlevel); #define GetTextWidth(hDlg, id) GetTextSize(GetDlgItem(hDlg, id), NULL).cx DWORD WINAPI HashThread(void* param); diff --git a/src/rufus.rc b/src/rufus.rc index 9a2845eb1c0..13274611c17 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2195" +CAPTION "Rufus 4.6.2196" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2195,0 - PRODUCTVERSION 4,6,2195,0 + FILEVERSION 4,6,2196,0 + PRODUCTVERSION 4,6,2196,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2195" + VALUE "FileVersion", "4.6.2196" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2195" + VALUE "ProductVersion", "4.6.2196" END END BLOCK "VarFileInfo" diff --git a/src/stdlg.c b/src/stdlg.c index 39e0e127f14..45c4673e5d7 100644 --- a/src/stdlg.c +++ b/src/stdlg.c @@ -46,7 +46,7 @@ /* Globals */ extern BOOL is_x86_64, appstore_version; -extern char unattend_username[MAX_USERNAME_LENGTH]; +extern char unattend_username[MAX_USERNAME_LENGTH], *sbat_level_txt; static HICON hMessageIcon = (HICON)INVALID_HANDLE_VALUE; static char* szMessageText = NULL; static char* szMessageTitle = NULL; @@ -1372,6 +1372,7 @@ static DWORD WINAPI CheckForFidoThread(LPVOID param) static BOOL is_active = FALSE; LONG_PTR style; char* loc = NULL; + uint32_t i; uint64_t len; HWND hCtrl; @@ -1382,6 +1383,19 @@ static DWORD WINAPI CheckForFidoThread(LPVOID param) return -1; is_active = TRUE; safe_free(fido_url); + safe_free(sbat_entries); + safe_free(sbat_level_txt); + + // Get the latest sbat_level.txt data while we're poking the network for Fido. + len = DownloadToFileOrBuffer(RUFUS_URL "/sbat_level.txt", NULL, (BYTE**)&sbat_level_txt, NULL, FALSE); + if (len != 0 && len < 512) { + sbat_entries = GetSbatEntries(sbat_level_txt); + if (sbat_entries != 0) { + for (i = 0; sbat_entries[i].product != NULL; i++); + if (i > 0) + uprintf("Found %d additional revoked UEFI bootloader filters from remote SBAT", i); + } + } // Get the Fido URL from parsing a 'Fido.ver' on our server. This enables the use of different // Fido versions from different versions of Rufus, if needed, as opposed to always downloading From 15c28434c0ca3594fdf63925da78daac3e4137a7 Mon Sep 17 00:00:00 2001 From: Pete Batard Date: Sat, 5 Oct 2024 01:05:55 +0100 Subject: [PATCH 15/30] [iso] add Microsoft SVN revocation validation and reporting * See https://github.com/pbatard/rufus/issues/2244#issuecomment-2243661539 * Note that we don't use the GUID but the resource name for bootmgr.efi, as trying to figure out where the heck the relevant GUID is located in the PE code is not worth the effort. * Also add internal fallback for sbat_level.txt if the user can't access our remote version. * Also improve PE section lookup. --- src/db.h | 12 +++++ src/hash.c | 83 +++++++++++++++++++++++++++------- src/net.c | 2 +- src/parser.c | 123 +++++++++++++++++++++++++++++++++++++++++++++++++-- src/rufus.c | 5 ++- src/rufus.h | 3 ++ src/rufus.rc | 10 ++--- src/stdlg.c | 2 +- 8 files changed, 210 insertions(+), 30 deletions(-) diff --git a/src/db.h b/src/db.h index 686980f61ed..30aa301ae4d 100644 --- a/src/db.h +++ b/src/db.h @@ -656,3 +656,15 @@ static uint8_t pe256dbx[] = { 0xff, 0xd7, 0x68, 0x8e, 0x7d, 0x2b, 0x8c, 0x3c, 0x31, 0x40, 0xb4, 0x15, 0xe7, 0x28, 0xbb, 0xe7, 0x66, 0x3c, 0x54, 0xe2, 0x3b, 0xd2, 0x88, 0xff, 0x2c, 0xf4, 0x61, 0x78, 0x35, 0x08, 0x8f, 0x39, 0xff, 0xf4, 0x21, 0xa9, 0xdc, 0xd3, 0xef, 0x38, 0xad, 0x58, 0x5e, 0x8b, 0xac, 0xa4, 0x08, 0xac, 0x2e, 0x4c, 0xdb, 0xdf, 0xa6, 0x79, 0x90, 0x0e, 0xc1, 0x70, 0x89, 0x62, 0x4e, 0x31, 0x0a, 0xda, }; + +/* + * Extended SBATLevel.txt that merges Linux SBAT with Microsoft's SVN + * See https://github.com/pbatard/rufus/issues/2244#issuecomment-2243661539 + * Use as fallback when https://rufus.ie/sbat_level.txt cannot be accessed. + */ +static const char db_sbat_level_txt[] = + "sbat,1,2024010900\n" + "shim,4\n" + "grub,3\n" + "grub.debian,4\n" + "BOOTMGRSECURITYVERSIONNUMBER,0x30000"; diff --git a/src/hash.c b/src/hash.c index 84ab9c5c6ea..1fd58c1ff0c 100644 --- a/src/hash.c +++ b/src/hash.c @@ -116,6 +116,7 @@ StrArray modified_files = { 0 }; extern int default_thread_priority; extern const char* efi_archname[ARCH_MAX]; +extern char* sbat_level_txt; /* * Rotate 32 or 64 bit integers by n bytes. @@ -2081,25 +2082,16 @@ BOOL IsFileInDB(const char* path) BOOL IsRevokedBySbat(uint8_t* buf, uint32_t len) { - static const char section_name[IMAGE_SIZEOF_SHORT_NAME] = { '.', 's', 'b', 'a', 't', '\0', '\0', '\0' }; - char* sbat = NULL, * version_str; - uint32_t i, j; + char* sbat = NULL, *version_str; + uint32_t i, j, sbat_len; sbat_entry_t entry; - IMAGE_SECTION_HEADER* section_header; - if (buf == NULL || len < 0x100 || sbat_entries == NULL) + if (sbat_entries == NULL) return FALSE; - for (i = 0x40; i < MIN(len, 0x800); i++) { - if (memcmp(section_name, &buf[i], sizeof(section_name)) == 0) { - section_header = (IMAGE_SECTION_HEADER*)&buf[i]; - if (section_header->SizeOfRawData >= len || section_header->PointerToRawData >= len) - return TRUE; - sbat = (char*)&buf[section_header->PointerToRawData]; - break; - } - } - if (sbat == NULL) + // Look for a .sbat section + sbat = GetPeSection(buf, &sbat_len, ".sbat"); + if (sbat == NULL || sbat < buf || sbat >= buf + len) return FALSE; for (i = 0; sbat[i] != '\0'; ) { @@ -2128,11 +2120,66 @@ BOOL IsRevokedBySbat(uint8_t* buf, uint32_t len) return FALSE; } +BOOL IsRevokedBySvn(uint8_t* buf, uint32_t len) +{ + wchar_t* rsrc_name = NULL; + uint8_t *base; + uint32_t i, j, rsrc_rva, rsrc_len, *svn_ver; + IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; + IMAGE_NT_HEADERS* pe_header; + IMAGE_NT_HEADERS64* pe64_header; + IMAGE_DATA_DIRECTORY img_data_dir; + + if (sbat_entries == NULL) + return FALSE; + + for (i = 0; sbat_entries[i].product != NULL; i++) { + // SVN entries are expected to be uppercase + for (j = 0; j < strlen(sbat_entries[i].product) && isupper(sbat_entries[i].product[j]); j++); + if (j < strlen(sbat_entries[i].product)) + continue; + rsrc_name = utf8_to_wchar(sbat_entries[i].product); + if (rsrc_name == NULL) + continue; + + pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + if (pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_I386 || pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_ARM) { + img_data_dir = pe_header->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_RESOURCE]; + } else { + pe64_header = (IMAGE_NT_HEADERS64*)pe_header; + img_data_dir = pe64_header->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_RESOURCE]; + } + + base = RvaToPhysical(buf, img_data_dir.VirtualAddress); + rsrc_rva = FindResourceRva(FALSE, base, base, rsrc_name, &rsrc_len); + safe_free(rsrc_name); + if (rsrc_rva != 0) { + if (rsrc_len == sizeof(uint32_t)) { + svn_ver = (uint32_t*)RvaToPhysical(buf, rsrc_rva); + if (svn_ver != NULL && *svn_ver < sbat_entries[i].version) + return TRUE; + } else { + uprintf("WARNING: Unexpected Microsoft SVN version size"); + } + } + } + return FALSE; +} + int IsBootloaderRevoked(uint8_t* buf, uint32_t len) { uint32_t i; uint8_t hash[SHA256_HASHSIZE]; + // Fall back to embedded sbat_level.txt if we couldn't access remote + if (sbat_entries == NULL) { + sbat_level_txt = safe_strdup(db_sbat_level_txt); + sbat_entries = GetSbatEntries(sbat_level_txt); + } + + // TODO: More elaborate PE checks? + if (buf == NULL || len < 0x100 || buf[0] != 'M' || buf[1] != 'Z') + return -2; if (!PE256Buffer(buf, len, hash)) return -1; // Check for UEFI DBX revocation @@ -2143,10 +2190,12 @@ int IsBootloaderRevoked(uint8_t* buf, uint32_t len) for (i = 0; i < pe256ssp_size * SHA256_HASHSIZE; i += SHA256_HASHSIZE) if (memcmp(hash, &pe256ssp[i], SHA256_HASHSIZE) == 0) return 2; - // Check for SBAT revocation + // Check for Linux SBAT revocation if (IsRevokedBySbat(buf, len)) return 3; - + // Sheck for Microsoft SVN revocation + if (IsRevokedBySvn(buf, len)) + return 4; return 0; } diff --git a/src/net.c b/src/net.c index 8e486ca08d6..f5fc821546e 100644 --- a/src/net.c +++ b/src/net.c @@ -240,7 +240,7 @@ uint64_t DownloadToFileOrBufferEx(const char* url, const char* file, const char* if (DownloadStatus != 200) { error_code = ERROR_INTERNET_ITEM_NOT_FOUND; SetLastError(RUFUS_ERROR(error_code)); - uprintf("%s: %d", (DownloadStatus == 404) ? "File not found" : "Unable to access file", DownloadStatus); + uprintf("%s '%s': %d", (DownloadStatus == 404) ? "File not found" : "Unable to access file", url, DownloadStatus); goto out; } dwSize = sizeof(strsize); diff --git a/src/parser.c b/src/parser.c index 2cb09ccf013..48905642dc0 100644 --- a/src/parser.c +++ b/src/parser.c @@ -1570,10 +1570,17 @@ sbat_entry_t* GetSbatEntries(char* sbatlevel) num_entries = 0; for (i = 0; sbatlevel[i] != '\0'; ) { - while (sbatlevel[i] == '\n') + // Eliminate blank lines + if (sbatlevel[i] == '\n') { i++; - if (sbatlevel[i] == '\0') - break; + continue; + } + // Eliminate comments + if (sbatlevel[i] == '#') { + while (sbatlevel[i] != '\n' && sbatlevel[i] != '\0') + i++; + continue; + } sbat_entries[num_entries].product = &sbatlevel[i]; for (; sbatlevel[i] != ',' && sbatlevel[i] != '\0' && sbatlevel[i] != '\n'; i++); if (sbatlevel[i] == '\0' || sbatlevel[i] == '\n') @@ -1586,7 +1593,11 @@ sbat_entry_t* GetSbatEntries(char* sbatlevel) sbatlevel[i] = '\0'; if (!eof) i++; - sbat_entries[num_entries].version = atoi(version_str); + // Allow the provision of an hex version + if (version_str[0] == '0' && version_str[1] == 'x') + sbat_entries[num_entries].version = strtoul(version_str, NULL, 16); + else + sbat_entries[num_entries].version = strtoul(version_str, NULL, 10); if (!eol) for (; sbatlevel[i] != '\0' && sbatlevel[i] != '\n'; i++); if (sbat_entries[num_entries].version != 0) @@ -1595,3 +1606,107 @@ sbat_entry_t* GetSbatEntries(char* sbatlevel) return sbat_entries; } + +/* + * PE parsing functions + */ + +// Return the address of a PE section from a PE buffer +uint8_t* GetPeSection(uint8_t* buf, uint32_t* sec_len, const char* name) +{ + char section_name[IMAGE_SIZEOF_SHORT_NAME] = { 0 }; + uint32_t i, nb_sections; + IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; + IMAGE_NT_HEADERS* pe_header; + IMAGE_NT_HEADERS64* pe64_header; + IMAGE_SECTION_HEADER* section_header; + + static_strcpy(section_name, name); + + pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + if (pe_header == NULL) + return NULL; + if (pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_I386 || pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_ARM) { + section_header = (IMAGE_SECTION_HEADER*)(&pe_header[1]); + nb_sections = pe_header->FileHeader.NumberOfSections; + } else { + pe64_header = (IMAGE_NT_HEADERS64*)pe_header; + section_header = (IMAGE_SECTION_HEADER*)(&pe64_header[1]); + nb_sections = pe64_header->FileHeader.NumberOfSections; + } + for (i = 0; i < nb_sections; i++) { + if (memcmp(section_header[i].Name, section_name, sizeof(section_name)) == 0) { + *sec_len = section_header->SizeOfRawData; + return &buf[section_header[i].PointerToRawData]; + } + } + return NULL; +} + +// Convert an RVA address to a physical address from a PE buffer +uint8_t* RvaToPhysical(uint8_t* buf, uint32_t rva) +{ + uint32_t i, nb_sections; + IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; + IMAGE_NT_HEADERS* pe_header; + IMAGE_NT_HEADERS64* pe64_header; + IMAGE_SECTION_HEADER* section_header; + + pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + if (pe_header == NULL) + return NULL; + + if (pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_I386 || pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_ARM) { + section_header = (IMAGE_SECTION_HEADER*)(pe_header + 1); + nb_sections = pe_header->FileHeader.NumberOfSections; + } else { + pe64_header = (IMAGE_NT_HEADERS64*)pe_header; + section_header = (IMAGE_SECTION_HEADER*)(pe64_header + 1); + nb_sections = pe64_header->FileHeader.NumberOfSections; + } + + for (i = 0; i < nb_sections; i++) { + if ((section_header[i].VirtualAddress <= rva && rva < (section_header[i].VirtualAddress + section_header[i].Misc.VirtualSize))) + break; + } + if (i >= nb_sections) + return NULL; + + return &buf[section_header[i].PointerToRawData + (rva - section_header[i].VirtualAddress)]; +} + +// Using the MS APIs to poke the resources of the EFI bootloaders is simply TOO. DAMN. SLOW. +// So, to QUICKLY access the resources we need, we reivent Microsoft's sub-optimal resource parser. +uint32_t FindResourceRva(BOOL found, uint8_t* base, uint8_t* cur, const wchar_t* name, uint32_t* len) +{ + uint32_t rva; + WORD i; + IMAGE_RESOURCE_DIRECTORY* dir = (IMAGE_RESOURCE_DIRECTORY*)cur; + IMAGE_RESOURCE_DIRECTORY_ENTRY* dir_entry = (IMAGE_RESOURCE_DIRECTORY_ENTRY*)&dir[1]; + IMAGE_RESOURCE_DIR_STRING_U* dir_string; + IMAGE_RESOURCE_DATA_ENTRY* data_entry; + + if (base == NULL || cur == NULL || name == NULL) + return 0; + + for (i = 0; i < dir->NumberOfNamedEntries + dir->NumberOfIdEntries; i++) { + if (!found && i < dir->NumberOfNamedEntries) { + dir_string = (IMAGE_RESOURCE_DIR_STRING_U*)(base + dir_entry[i].NameOffset); + if (dir_string->Length != wcslen(name) || + memcmp(name, dir_string->NameString, wcslen(name)) != 0) + continue; + found = TRUE; + } + if (dir_entry[i].OffsetToData & IMAGE_RESOURCE_DATA_IS_DIRECTORY) { + rva = FindResourceRva(found, base, &base[dir_entry[i].OffsetToDirectory], name, len); + if (rva != 0) + return rva; + } else if (found) { + data_entry = (IMAGE_RESOURCE_DATA_ENTRY*)(base + dir_entry[i].OffsetToData); + if (len != NULL) + *len = data_entry->Size; + return data_entry->OffsetToData; + } + } + return 0; +} diff --git a/src/rufus.c b/src/rufus.c index bb4647c01d4..cdf10b8654f 100755 --- a/src/rufus.c +++ b/src/rufus.c @@ -1608,6 +1608,7 @@ static DWORD WINAPI BootCheckThread(LPVOID param) const char* msg; for (i = 0; i < ARRAYSIZE(img_report.efi_boot_path) && img_report.efi_boot_path[i][0] != 0; i++) { + static const char* revocation_type[] = { "UEFI DBX", "Windows SecuritySiPolicy", "Linux SBAT", "Windows SVN" }; len = ReadISOFileToBuffer(image_path, img_report.efi_boot_path[i], &buf); if (len == 0) { uprintf("Warning: Failed to extract '%s' to check for UEFI revocation", img_report.efi_boot_path[i]); @@ -1616,8 +1617,8 @@ static DWORD WINAPI BootCheckThread(LPVOID param) r = IsBootloaderRevoked(buf, len); safe_free(buf); if (r > 0) { - uprintf("Warning: '%s' is revoked by %s", img_report.efi_boot_path[i], - (r == 1) ? "UEFI DBX" : ((r == 2) ? "Windows SSP" : "Linux SBAT")); + assert(r <= ARRAYSIZE(revocation_type)); + uprintf("Warning: '%s' has been revoked by %s", img_report.efi_boot_path[i], revocation_type[r - 1]); is_bootloader_revoked = TRUE; switch (r) { case 2: diff --git a/src/rufus.h b/src/rufus.h index 3fafca6601a..0ff7f209848 100644 --- a/src/rufus.h +++ b/src/rufus.h @@ -831,6 +831,9 @@ extern HANDLE CreatePreallocatedFile(const char* lpFileName, DWORD dwDesiredAcce DWORD dwFlagsAndAttributes, LONGLONG fileSize); extern uint32_t ResolveDllAddress(dll_resolver_t* resolver); extern sbat_entry_t* GetSbatEntries(char* sbatlevel); +extern uint8_t* GetPeSection(uint8_t* buf, uint32_t* sec_len, const char* name); +extern uint8_t* RvaToPhysical(uint8_t* buf, uint32_t rva); +extern uint32_t FindResourceRva(BOOL found, uint8_t* base, uint8_t* cur, const wchar_t* name, uint32_t* len); #define GetTextWidth(hDlg, id) GetTextSize(GetDlgItem(hDlg, id), NULL).cx DWORD WINAPI HashThread(void* param); diff --git a/src/rufus.rc b/src/rufus.rc index 13274611c17..ec45982a6d8 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2196" +CAPTION "Rufus 4.6.2197" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2196,0 - PRODUCTVERSION 4,6,2196,0 + FILEVERSION 4,6,2197,0 + PRODUCTVERSION 4,6,2197,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2196" + VALUE "FileVersion", "4.6.2197" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2196" + VALUE "ProductVersion", "4.6.2197" END END BLOCK "VarFileInfo" diff --git a/src/stdlg.c b/src/stdlg.c index 45c4673e5d7..0be692459cf 100644 --- a/src/stdlg.c +++ b/src/stdlg.c @@ -1393,7 +1393,7 @@ static DWORD WINAPI CheckForFidoThread(LPVOID param) if (sbat_entries != 0) { for (i = 0; sbat_entries[i].product != NULL; i++); if (i > 0) - uprintf("Found %d additional revoked UEFI bootloader filters from remote SBAT", i); + uprintf("Found %d additional UEFI revocation filters from remote SBAT", i); } } From 3e840a94ce33840c45c503d8cb113ac0b1746dfb Mon Sep 17 00:00:00 2001 From: Pete Batard Date: Sat, 5 Oct 2024 11:03:42 +0100 Subject: [PATCH 16/30] [wue] set password not to expire when creating a local account * Looks like using the 'net user USERNAME /logonpasswordchg:yes" might have the side effect of setting the main user account to expire after a few months. So to alleviate that, we'll just set the system policy to use passwords that never expire. * Also clean up the PE parsing code and fix 2 Coverity warnings. * Also fix typos, misprints and ditch the UNRELIABLE timestamp.acs.microsoft.com server. --- _sign.cmd | 2 +- res/appstore/packme.cmd | 2 +- res/hogger/hogger.asm | 2 +- res/hogger/hogger.c | 2 +- src/hash.c | 10 +++++----- src/parser.c | 43 ++++++++++++++++++++++++----------------- src/rufus.c | 2 +- src/rufus.h | 4 ++-- src/rufus.rc | 10 +++++----- src/stdfn.c | 2 +- src/wue.c | 6 ++++++ 11 files changed, 49 insertions(+), 36 deletions(-) diff --git a/_sign.cmd b/_sign.cmd index 1da906b6545..40f24e2203d 100644 --- a/_sign.cmd +++ b/_sign.cmd @@ -1,2 +1,2 @@ @echo off -"C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64\signtool" sign /v /sha1 fc4686753937a93fdcd48c2bb4375e239af92dcb /fd SHA256 /tr http://timestamp.acs.microsoft.com /td SHA256 %* +"C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64\signtool" sign /v /sha1 fc4686753937a93fdcd48c2bb4375e239af92dcb /fd SHA256 /tr http://timestamp.digicert.com /td SHA256 %* diff --git a/res/appstore/packme.cmd b/res/appstore/packme.cmd index 193124aaaef..bf7a62eeede 100644 --- a/res/appstore/packme.cmd +++ b/res/appstore/packme.cmd @@ -98,7 +98,7 @@ if "%VERSION_OVERRIDE%"=="" ( echo Will create %VERSION% AppStore Bundle pause -"%WDK_PATH%\signtool" sign /v /sha1 %SIGNATURE_SHA1% /fd SHA256 /tr http://timestamp.acs.microsoft.com /td SHA256 *.exe +"%WDK_PATH%\signtool" sign /v /sha1 %SIGNATURE_SHA1% /fd SHA256 /tr http://timestamp.digicert.com /td SHA256 *.exe if ERRORLEVEL 1 goto out echo [Files]> bundle.map diff --git a/res/hogger/hogger.asm b/res/hogger/hogger.asm index dbab990c42d..836b8e920f8 100644 --- a/res/hogger/hogger.asm +++ b/res/hogger/hogger.asm @@ -1,6 +1,6 @@ ; Rufus: The Reliable USB Formatting Utility ; Commandline hogger, assembly version (NASM) - ; Copyright 2014 Pete Batard + ; Copyright © 2014 Pete Batard ; ; This program is free software: you can redistribute it and/or modify ; it under the terms of the GNU General Public License as published by diff --git a/res/hogger/hogger.c b/res/hogger/hogger.c index a03b76eedc6..01164b3795c 100644 --- a/res/hogger/hogger.c +++ b/res/hogger/hogger.c @@ -1,7 +1,7 @@ /* * Rufus: The Reliable USB Formatting Utility * Commandline hogger, C version - * Copyright 2014 Pete Batard + * Copyright © 2014 Pete Batard * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/hash.c b/src/hash.c index 1fd58c1ff0c..c5dd18f85bb 100644 --- a/src/hash.c +++ b/src/hash.c @@ -2090,7 +2090,7 @@ BOOL IsRevokedBySbat(uint8_t* buf, uint32_t len) return FALSE; // Look for a .sbat section - sbat = GetPeSection(buf, &sbat_len, ".sbat"); + sbat = GetPeSection(buf, ".sbat", &sbat_len); if (sbat == NULL || sbat < buf || sbat >= buf + len) return FALSE; @@ -2123,7 +2123,7 @@ BOOL IsRevokedBySbat(uint8_t* buf, uint32_t len) BOOL IsRevokedBySvn(uint8_t* buf, uint32_t len) { wchar_t* rsrc_name = NULL; - uint8_t *base; + uint8_t *root; uint32_t i, j, rsrc_rva, rsrc_len, *svn_ver; IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; IMAGE_NT_HEADERS* pe_header; @@ -2150,8 +2150,8 @@ BOOL IsRevokedBySvn(uint8_t* buf, uint32_t len) img_data_dir = pe64_header->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_RESOURCE]; } - base = RvaToPhysical(buf, img_data_dir.VirtualAddress); - rsrc_rva = FindResourceRva(FALSE, base, base, rsrc_name, &rsrc_len); + root = RvaToPhysical(buf, img_data_dir.VirtualAddress); + rsrc_rva = FindResourceRva(rsrc_name, root, root, &rsrc_len); safe_free(rsrc_name); if (rsrc_rva != 0) { if (rsrc_len == sizeof(uint32_t)) { @@ -2159,7 +2159,7 @@ BOOL IsRevokedBySvn(uint8_t* buf, uint32_t len) if (svn_ver != NULL && *svn_ver < sbat_entries[i].version) return TRUE; } else { - uprintf("WARNING: Unexpected Microsoft SVN version size"); + uprintf("WARNING: Unexpected Secure Version Number size"); } } } diff --git a/src/parser.c b/src/parser.c index 48905642dc0..2f000650464 100644 --- a/src/parser.c +++ b/src/parser.c @@ -1611,8 +1611,8 @@ sbat_entry_t* GetSbatEntries(char* sbatlevel) * PE parsing functions */ -// Return the address of a PE section from a PE buffer -uint8_t* GetPeSection(uint8_t* buf, uint32_t* sec_len, const char* name) +// Return the address and (optionally) the length of a PE section from a PE buffer +uint8_t* GetPeSection(uint8_t* buf, const char* name, uint32_t* len) { char section_name[IMAGE_SIZEOF_SHORT_NAME] = { 0 }; uint32_t i, nb_sections; @@ -1623,9 +1623,10 @@ uint8_t* GetPeSection(uint8_t* buf, uint32_t* sec_len, const char* name) static_strcpy(section_name, name); - pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; - if (pe_header == NULL) + if (buf == NULL || name == NULL) return NULL; + + pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; if (pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_I386 || pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_ARM) { section_header = (IMAGE_SECTION_HEADER*)(&pe_header[1]); nb_sections = pe_header->FileHeader.NumberOfSections; @@ -1636,7 +1637,8 @@ uint8_t* GetPeSection(uint8_t* buf, uint32_t* sec_len, const char* name) } for (i = 0; i < nb_sections; i++) { if (memcmp(section_header[i].Name, section_name, sizeof(section_name)) == 0) { - *sec_len = section_header->SizeOfRawData; + if (len != NULL) + *len = section_header->SizeOfRawData; return &buf[section_header[i].PointerToRawData]; } } @@ -1652,10 +1654,10 @@ uint8_t* RvaToPhysical(uint8_t* buf, uint32_t rva) IMAGE_NT_HEADERS64* pe64_header; IMAGE_SECTION_HEADER* section_header; - pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; - if (pe_header == NULL) + if (buf == NULL) return NULL; + pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; if (pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_I386 || pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_ARM) { section_header = (IMAGE_SECTION_HEADER*)(pe_header + 1); nb_sections = pe_header->FileHeader.NumberOfSections; @@ -1677,32 +1679,37 @@ uint8_t* RvaToPhysical(uint8_t* buf, uint32_t rva) // Using the MS APIs to poke the resources of the EFI bootloaders is simply TOO. DAMN. SLOW. // So, to QUICKLY access the resources we need, we reivent Microsoft's sub-optimal resource parser. -uint32_t FindResourceRva(BOOL found, uint8_t* base, uint8_t* cur, const wchar_t* name, uint32_t* len) +static BOOL FoundResourceRva = FALSE; +uint32_t FindResourceRva(const wchar_t* name, uint8_t* root, uint8_t* dir, uint32_t* len) { uint32_t rva; WORD i; - IMAGE_RESOURCE_DIRECTORY* dir = (IMAGE_RESOURCE_DIRECTORY*)cur; - IMAGE_RESOURCE_DIRECTORY_ENTRY* dir_entry = (IMAGE_RESOURCE_DIRECTORY_ENTRY*)&dir[1]; + IMAGE_RESOURCE_DIRECTORY* _dir = (IMAGE_RESOURCE_DIRECTORY*)dir; + IMAGE_RESOURCE_DIRECTORY_ENTRY* dir_entry = (IMAGE_RESOURCE_DIRECTORY_ENTRY*)&_dir[1]; IMAGE_RESOURCE_DIR_STRING_U* dir_string; IMAGE_RESOURCE_DATA_ENTRY* data_entry; - if (base == NULL || cur == NULL || name == NULL) + if (root == NULL || dir == NULL || name == NULL) return 0; - for (i = 0; i < dir->NumberOfNamedEntries + dir->NumberOfIdEntries; i++) { - if (!found && i < dir->NumberOfNamedEntries) { - dir_string = (IMAGE_RESOURCE_DIR_STRING_U*)(base + dir_entry[i].NameOffset); + // Initial invocation should always start at the root + if (root == dir) + FoundResourceRva = FALSE; + + for (i = 0; i < _dir->NumberOfNamedEntries + _dir->NumberOfIdEntries; i++) { + if (!FoundResourceRva && i < _dir->NumberOfNamedEntries) { + dir_string = (IMAGE_RESOURCE_DIR_STRING_U*)(root + dir_entry[i].NameOffset); if (dir_string->Length != wcslen(name) || memcmp(name, dir_string->NameString, wcslen(name)) != 0) continue; - found = TRUE; + FoundResourceRva = TRUE; } if (dir_entry[i].OffsetToData & IMAGE_RESOURCE_DATA_IS_DIRECTORY) { - rva = FindResourceRva(found, base, &base[dir_entry[i].OffsetToDirectory], name, len); + rva = FindResourceRva(name, root, &root[dir_entry[i].OffsetToDirectory], len); if (rva != 0) return rva; - } else if (found) { - data_entry = (IMAGE_RESOURCE_DATA_ENTRY*)(base + dir_entry[i].OffsetToData); + } else if (FoundResourceRva) { + data_entry = (IMAGE_RESOURCE_DATA_ENTRY*)(root + dir_entry[i].OffsetToData); if (len != NULL) *len = data_entry->Size; return data_entry->OffsetToData; diff --git a/src/rufus.c b/src/rufus.c index cdf10b8654f..dade331f510 100755 --- a/src/rufus.c +++ b/src/rufus.c @@ -1608,7 +1608,7 @@ static DWORD WINAPI BootCheckThread(LPVOID param) const char* msg; for (i = 0; i < ARRAYSIZE(img_report.efi_boot_path) && img_report.efi_boot_path[i][0] != 0; i++) { - static const char* revocation_type[] = { "UEFI DBX", "Windows SecuritySiPolicy", "Linux SBAT", "Windows SVN" }; + static const char* revocation_type[] = { "UEFI DBX", "Windows SSP", "Linux SBAT", "Windows SVN" }; len = ReadISOFileToBuffer(image_path, img_report.efi_boot_path[i], &buf); if (len == 0) { uprintf("Warning: Failed to extract '%s' to check for UEFI revocation", img_report.efi_boot_path[i]); diff --git a/src/rufus.h b/src/rufus.h index 0ff7f209848..389fb12bcf1 100644 --- a/src/rufus.h +++ b/src/rufus.h @@ -831,9 +831,9 @@ extern HANDLE CreatePreallocatedFile(const char* lpFileName, DWORD dwDesiredAcce DWORD dwFlagsAndAttributes, LONGLONG fileSize); extern uint32_t ResolveDllAddress(dll_resolver_t* resolver); extern sbat_entry_t* GetSbatEntries(char* sbatlevel); -extern uint8_t* GetPeSection(uint8_t* buf, uint32_t* sec_len, const char* name); +extern uint8_t* GetPeSection(uint8_t* buf, const char* name, uint32_t* len); extern uint8_t* RvaToPhysical(uint8_t* buf, uint32_t rva); -extern uint32_t FindResourceRva(BOOL found, uint8_t* base, uint8_t* cur, const wchar_t* name, uint32_t* len); +extern uint32_t FindResourceRva(const wchar_t* name, uint8_t* root, uint8_t* dir, uint32_t* len); #define GetTextWidth(hDlg, id) GetTextSize(GetDlgItem(hDlg, id), NULL).cx DWORD WINAPI HashThread(void* param); diff --git a/src/rufus.rc b/src/rufus.rc index ec45982a6d8..ffba8ce100a 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2197" +CAPTION "Rufus 4.6.2198" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2197,0 - PRODUCTVERSION 4,6,2197,0 + FILEVERSION 4,6,2198,0 + PRODUCTVERSION 4,6,2198,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2197" + VALUE "FileVersion", "4.6.2198" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2197" + VALUE "ProductVersion", "4.6.2198" END END BLOCK "VarFileInfo" diff --git a/src/stdfn.c b/src/stdfn.c index 5265bf94d7d..ec4933a03ca 100644 --- a/src/stdfn.c +++ b/src/stdfn.c @@ -738,7 +738,7 @@ BOOL FileIO(enum file_io_type io_type, char* path, char** buffer, DWORD* size) /* * Get a resource from the RC. If needed that resource can be duplicated. * If duplicate is true and len is non-zero, the a zeroed buffer of 'len' - * size is allocated for the resource. Else the buffer is allocate for + * size is allocated for the resource. Else the buffer is allocated for * the resource size. */ uint8_t* GetResource(HMODULE module, char* name, char* type, const char* desc, DWORD* len, BOOL duplicate) diff --git a/src/wue.c b/src/wue.c index 9fadd53d9b2..197ed5a76ee 100644 --- a/src/wue.c +++ b/src/wue.c @@ -194,6 +194,12 @@ char* CreateUnattendXml(int arch, int flags) fprintf(fd, " %d\n", order++); fprintf(fd, " net user "%s" /logonpasswordchg:yes\n", unattend_username); fprintf(fd, " \n"); + // Some people report that using the `net user` command above might reset the password expiration to 90 days... + // To alleviate that, blanket set passwords on the target machine to never expire. + fprintf(fd, " \n"); + fprintf(fd, " %d\n", order++); + fprintf(fd, " net accounts /maxpwage:unlimited\n"); + fprintf(fd, " \n"); fprintf(fd, " \n"); } } From 98a42a235f67f6e535b6ab07c23bd4eb5f8ba770 Mon Sep 17 00:00:00 2001 From: Pete Batard Date: Sat, 5 Oct 2024 20:00:40 +0100 Subject: [PATCH 17/30] [wue] add setup wrapper project for Windows 11 24H2 in-place upgrades --- .github/workflows/setup.yml | 52 ++++++ res/icons/license.txt | 5 + res/icons/setup.ico | Bin 0 -> 31354 bytes res/icons/setup.svg | 11 ++ res/setup/.editorconfig | 9 ++ res/setup/readme.txt | 29 ++++ res/setup/resource.h | 16 ++ res/setup/setup.c | 125 +++++++++++++++ res/setup/setup.rc | 116 ++++++++++++++ res/setup/setup.sln | 37 +++++ res/setup/setup.vcxproj | 272 ++++++++++++++++++++++++++++++++ res/setup/setup.vcxproj.filters | 27 ++++ src/rufus.rc | 12 +- 13 files changed, 705 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/setup.yml create mode 100644 res/icons/setup.ico create mode 100644 res/icons/setup.svg create mode 100644 res/setup/.editorconfig create mode 100644 res/setup/readme.txt create mode 100644 res/setup/resource.h create mode 100644 res/setup/setup.c create mode 100644 res/setup/setup.rc create mode 100644 res/setup/setup.sln create mode 100644 res/setup/setup.vcxproj create mode 100644 res/setup/setup.vcxproj.filters diff --git a/.github/workflows/setup.yml b/.github/workflows/setup.yml new file mode 100644 index 00000000000..98c62dc3d70 --- /dev/null +++ b/.github/workflows/setup.yml @@ -0,0 +1,52 @@ +name: setup +run-name: Windows setup wrapper build + +on: + workflow_dispatch: + branches: [ master ] + +env: + SOLUTION_FILE_PATH: ./res/setup/setup.sln + BUILD_CONFIGURATION: Release + +jobs: + VS2022-Build: + runs-on: windows-latest + + strategy: + matrix: + TARGET_PLATFORM: [x64, arm64] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Add MSBuild to PATH + uses: microsoft/setup-msbuild@v2 + with: + msbuild-architecture: x64 + + - name: Build + shell: cmd + run: | + msbuild ${{ env.SOLUTION_FILE_PATH }} /m /p:Configuration=${{ env.BUILD_CONFIGURATION }},Platform=${{ matrix.TARGET_PLATFORM }} + move .\res\setup\${{ matrix.TARGET_PLATFORM }}\Release\setup.exe .\setup_${{ matrix.TARGET_PLATFORM }}.exe + + - name: Display SHA-256 + run: sha256sum ./setup_${{ matrix.TARGET_PLATFORM }}.exe + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.TARGET_PLATFORM }} + path: ./*.exe + + Extra-Step-To-Merge-Artifacts-Thanks-To-Upload-Artifact-v4-Breaking-Backwards-Compatibility: + runs-on: windows-latest + needs: VS2022-Build + steps: + - name: Merge Artifacts + uses: actions/upload-artifact/merge@v4 + with: + name: setup + delete-merged: true diff --git a/res/icons/license.txt b/res/icons/license.txt index 4d78ab299ab..cd9f039c16b 100644 --- a/res/icons/license.txt +++ b/res/icons/license.txt @@ -8,6 +8,11 @@ o hash-*.png, info-*.png, lang-*.png, log-*.png, save-*.png, settings-*.png CC BY-ND 4.0 when used as native resolution bitmaps, Commercial License otherwise See https://www.axialis.com/icongenerator/iconset-license.html#free +o setup.ico + Base SVG from https://tablericons.com/ + No attribution required. + With arrow overlay from Axialis Icon Set. + NB: To be on the safe side with regards to icon use and redistribution, we did purchase a Commercial License for the Axialis icons. If you are planning to use these icons outside of this project, you may diff --git a/res/icons/setup.ico b/res/icons/setup.ico new file mode 100644 index 0000000000000000000000000000000000000000..fcfba3ec3259ddabc4e69d216f9558cf7536940a GIT binary patch literal 31354 zcmeIb1z1&C+dsTHv^1!6NTVX%DWM<=5`rj5my}3Ja}WWMMi8V+LXlFifa8E5QVNQM z#E8<}NStqN;bs<_hS~F!CUOKhIGz(N&#W)_oD)*f)7?OGa3B$}yOG^CFTn)!LUM70$ynZ{#6v*pHol!TFu>1AIj= zKMXSucE1akS^Y3&xNf)X59DE=*8o0Xd>2X!{X>sBFb_|McyNDdlwwLgUG!iuQ)I2C_rNj=mUkC8c zp2NPKHf8u0KYO4H*v9CaI%Cbm9deFOB*j`3{(jN{FyBx0wX;XyPkigQt>e4*2fr6g z{jX9q8iH1^AjlsezypF(cOZG24gngpx&wd?IvWTHWMLzqtAgN{4EI0q&lvPs|JQzg zI)_POO@$7DZ$mB9R{V6FsSwpa@(jM(hlpYMI}Va+;5}Ug2n8qrXaE=mSOfjU21wVp zF$2=mKeCa3Nm2pST?hCk58B&0ERO-GTLx$ccmQw{z%|QG{0!)CMN+IpMM1e1U^d=N zc<)XdfEIe!NgDQH9^fs&H+j%b&m>p~ivU_3K;=%k-T7oIQJ8NX#OvELL>mk80NNTr zT#)_|yxlUGGOQas$1nA}sq=8a`ZMjs&tl}_%!GB}=TEc{;la>;l)}3IO26|-ZVp&i zrk%u2{bymD{yK-jsQUBY-|-*T{{#R3$MwU{5T77@z8}R$3B=6U--35* z{NDWr3NA~r1K)E2CbzNmo-xm}E3JpWnVs(!*5`>lgb(`cO8~C`7PkTCFL?wU#7}n> zhvx>bVf~o*45vTH*8|#j?bFhSodM{1v)%V^fcy~14}lcU!+7Hy$jy)eYykbayPt1o zIY5v1yZD^7WrF?Q(v2A(Ej-!fj9pmeFy(E#Z{X*EW3^M??)>id4|KyZ+46H+&&Gwz z2jq0EndNrVVcy;Pc9(aze;@;H|c@0mkSXFfY&*4dx1RV2p<26AjbgI*{K0a z!umAL7>G}tqa%j%Fb(L1>);svWqgvXL(fww!9|4+ORgX4IhkrnyHmv(w zeGr#pqMHaVkbeR2ZF~T8$unRsB9d+^CIskD0T#jdLJRBAwqS(Y=WE!%?HI4xUE|cX z3*^uRwACH-*XJBq@PRBhz_;-Y6pdi541-51}D!XaLNV zQ24V@@bD}Y`6GbI7z*(C10Dzl*&iT}oP`KM84uJ$K$kx=K7^7(Z_UG407V`EGY|78 z2ZjIs{>LIva7k((Xru5vNC&_L;3hyBz)OI;0BZk=aUj22e=MNE`r85Kz!>svKKQdd z;EVW`4`AI)2zX8c!2bU|{b1aK?E(IAfJOlLS;GMA08WDWD?ItBI9#{W~`z4C%N=AZOq z?C;j~vvR>DN#Sk%4ZC@Oy`=5BR z?*c79g=x7xpqu`m5!`#%f#F8cp-@pqD50`2f`?UTTv_(z{V*HQAP-yhI* z1$=|_yF&b@V*X_*dcX^>dH%Am{~^!UzWPt!KgJN4v+{v?Qyf4Cm`g2h0nP&&T!-NU zx@`cy{Lsd)eEW%gFgGLsHf}ivbFE|MoqtZb*0*@BaS_0NZ2Pc*@#(M99ETsr-!5>= zcGaKlal?C$-}WCd^}vTJ$3J}d<+^RBPe1E_zdrZZxj4oKoML2m(lB{g2Cj$O1BUOf z(jVCe`-^-(i#bLg#^%lxv~~;z{!aQs``{bH=mh!}0Q}%@4yIvr!ujp*&=2PM#vjbRjN$n^ z=}+u0Vf5L9^%<-O*iZxbrW?+~G`J4t!SDm0mSEc7{ed;;pZY(b_gnlyz7=3=e-fms z7`>Q0TnB8nezX4%#vjb_gtq!Cum`(C_f3pGI0gFQHS0c1Jtl=|aNT#jL0mWCdeCo^ zgL(6x=7$(rObYnSVEZ5~#n9mI59s}_&jaN)U~fN$2hPK^?`#7ZOn(gYWo_%l%#ZH@ z+8^`RuWJ%(K%3g;`)(Z#^n-r)`~C;e-r5FoP}^Q*htEFV{-FMA|KlXN2k31A_`BY=`g@<|D=I03?M$g+$#q> zfLOyqwf{=nuA|E2%W`48Sv z3FbZhtMUI|X+QtRD$5I^;Qx6@!xng#63xdA)^_y_>+SHpnwFb&jkvh0=vzFGln&0}G||IPSI z|H0lbGw>}3;M;vDc)t;px9&_~_8#Ff*n5O|a$uRA&jDht4L<`Mdw~B)`+@C)eM)`c z&&-x@f84j)y3h9)v>8}-=Q9BR;r9T?|6iW}Z|i||{WSi-zsuWux!>Xp`@9|RK9Gm^ zqG9yGv>kO@`EA+DJD&l>J_q1m_J8|5!r##k{Rh72!}fpV7fb{A<^#xYon7wse@ix) z8{W(Q_8x#;_?x!d`Aq+49{Bvr@e6!|aC`r&HuKGYzzd&u{i;m?U4MKJz&`j(sDJ(k z`wxKW2mTa);L9D@pY1sRsa0$T##A2y(8 zZr6SDACxieKNI-Ab+%v&WB`41oA;}}fAAdtXHWMR?f<*>1MCk2eA{QYzs3bWLps1t z|9xQ^T!*m<;?E0c!#m$4DF4s5pCGP^oi?}gBfvKAAHHq92lzO1>j&!iF>!|pFfaH9 z{@i|m?ynO6+JC`)e09wH0jvSO`&&t1+ZAAE`@!T#fluFj{+S0b55V{B<_~?YKa=}M z=%D?$17ODA#GSHRbNrw6AFyA+Tmn1!cjnH>y@PqzI1n^G)VA|U5 z6kZE3VfcaloojhaJNQS_d2l`vo zANjv6``!9ww~gIp5Yr!?U&H+j@H-yh4B+|c-+1oUxAh)=-IjK@6-*g!f8WJ@x9?C;t#k$={c0b=-%3$5*juST4OTVmCrHt73jUb>Z>0zt`7J(d z8q`*bmFWLf4*oMUSdTA+g5%&XjpYyf1N8gDaYBLXSa941Abd0&#~BD*$^yY#fta__ zO~`L6y?$E#ILRKmJwOwQs)~{pd|L|!0sO@qT+Xuu368UZsty78c@vn2KoAS0s&rK6 zO7di-hj;QjpK0`_bPR`nDM!(qL!&79;aI$rod-m130|`sCk5VQXFqUiPeny_(gNqC zeNxoqW;bWXDSB%vmbFkP7DDp1(G*?x~SzIgYL@7n&dGqiR zNSbgs2{>8m5OM5Q5hqg0srg^~cUg2Eoh!pS8=S9r!9qLN;r(ywOj2faMi-9HcQ;QM zR>nv4hD&&nA&_>NeMT`xnWdD+(iS+6rHMz*KDfY?9c2}c|bv#dYZL+s~I>trH5;qeYmGCGhJm_7~ zY+WdqVi60!ISnP^w2uDs1n2$}?~)>Ou6%m=wwH?EIZ75`z-DtIwRwWNyaLhxyo$u8 z(CLc@-39Ns-fFLia+01oTl4gRhP#lG1xJ|7ID1{{s#V^Dn=byR+7t#gj)fk<%Jl9| z!2Q@cTd81#D;E<;L@Y|uTG$hI;NZ%JmBE{qxW=JNN;y8~uE+MfGii_vp3}~luAG)4@ zh_HuyweHp6nD@?a$nVdg5i|trWGZnu!>4_I@t-{i8ru;^yz} z0vG!}EO`6!tQdu6j#RUsdil;Z_)tvU@izRhLq{W3T;jtRPrYlHx9*+sOyD8-6qvjK z9cxLgdvKSJgHAs*=IQ2aj4%6Wt%1R`bNBOzF*|av{4QTd9`iF2MSW6E= zRD7~G#%{Ep!2QC2+|+NYRz27}h6BxAj9!ywy}rnM}0;KMPi#0e@t}4L&8r z;B|#+CWl_%LCGeMeD?v9jN9Jb?M8M*g0gyfecbJNBI%uYkgm zWx{FSUx;oQ<-=!+7ruw^Eq{+S96fd{)*A`RQhvIN%sm=0>x|8H6W$|)Zn6vJRn21` zjq#^FxTY^OEy@!smy(SSCP2p1FBo))d>_%q`gCqPGh?0XPW_nXqSrN{2#vT48KDo1688l6Im_M9q zFL4V!dZq^oevcw;nQ@+}D&s46ytyi!xsOYTut@fj zP>-IBU|!IIKBOZVK+9YiP@4a4H0K%m@Tm4K8b>QD)?PMp_WZFN<>BsWhbCiphE z)Yd4}e`QzA_soO@I5!%_SpsH3o_Yi2QxG%shYTbaT6jqC9z z9=Z2bvOiT8WE@|RejKZIyWuqLRWf@?d%uVe-W14~S%|wVdbW?&$<)PP~ak zaVUhzO$&prQ*>sjR!g}c_JLNLJBN>YTQEn;A%R;1>-x%VPjRZt6aCXOp9E3Ge~{O- z@lY@7xeMtSd5bEuHMRvC#Vo`ppS!7SSHwDGV04r>-+^JgxcB7QY!U1RPF87)M z^T!XXa8%zry!t$B(e`1oTv0Xis8ZW?O0u5Rz;x4(ha)Oyo+MwG54nqVye0M6VjADc z_ajDmDrgadx{Y$92R>EVlTlCR`#r6PrzLZ(wT{T?NLp-s>@5GQts z&vfEL#n7j7!IM~FhF1i$G!l{`zW6yjIz0Js@mTP`2PH3Plk!i>~{(RrS`1$bvk3`oM7Q%aDE z2S(YA!v{1poe0_L)h+4|MOrS>1y8g^g85JAtH!86Mn4bnn^WZ*VUtAcleKrsI7Eh8 z)2nDMAHfqu*caK|eoU9yURC@6O8di?35`T;BSDsex6J2`*+sXD2z2jpcHt;-8 z?{Y;E<42w2Z$u5}cHAJ-XOF-}liKu#l*9=a^$NeffTA1lL%fg>c7JKoj@3sXO}5@K z!#_h`qm62m$F{y1v*)bD(KeNVQ>aK~L7|y1L|x|>T%*i19<7|LOL4yJIf2g0ZZC6t z$Y8w2)8mNmmaA)k@U2!T_TL>!+K`a zw%&I^j@-_OSemBi-ea_-+r5^S+ejt^d0ZU-nxmY`>`TF@z5}DF_Y63W*RBwDQPVFO zkJvew4BET5RoPpq-=_wQ8*I~=c4O>XNl9lU+I%P!y)+biZUy*#7Pka!i?-)x7RIX- zo%C^05~x=Ui3x{DhX-8a$2?sxx$j#?q}#W;ouhX@ZE#NgMQxX1pEJtP<-m--DpA>L zaHns_5)aFC?}d71zndu*;xcy3?o&J`9}O&gnsStry*d8nh0}V%tQ5Nlj~#d63uS%ftLs=(k5&BYK*9Q=;36to#R8RI}S-dxJzvM$4tvyr(9V-}Bo)&l#<` zh0J(xx&XVZ>rsTcjXPs;-)SywtB8RCbvcHCf&7oD4TEmobZ+dC;rM<<2RWO-SEijH z>_?Fk_f7`yRnQ?^^bs4a^36{1r8ATjJwuO~y$>E*#PcFRO=$S~$2gr~sXW#8D(rkK zh0}-CwazLfEw-SDCEfB5m-W(yR?Cj+ocrxmmZfpp_dE$RCs7;E8SjfJpW1M z&gzop?jrAs))N1=SJ#s-Pd1O%@FD!t4$+gb>dRH5N&9-An^aDQSN1+{Pj#3s`q0x| zT-h`DJL5As9rYrMp;5K`V9k7H@E0ia2%g-9U3?YCF9lg8yQl z-JJ_Q`kAIfN0i@42|38%cWxFk8ijVzOIh~!>6ZI04@-M(J^0iRl@b)wp*0?%eg%HH07GgX3aQ|BdyGRJ8&&qX(7J}1avR!bb#>gEE#Ztky2ikM6+@?j=8$oTxzve9pO z`t7Ip9>_=c~}1K!a`vZ>u>#0~~r5Bl6v zvM0o^RHS7nXxd0PK3Q<26`BuZzhFnL_C&jlBeC3m&2pfq#KWYEMA+^^il$eOQI``*q3H>ybLF%W613Brw`Q| zxX8SFNTgkPG4j|OGs<~EoAxE$30f9M-yw%F-}%ZgyTf=iXQadmb(CZ7H{`uwFkUC0 zIie(#eZr()P}r@`sYkY)*N;0tA$B72HO~6uS-wEQa}l?&@0a*oRT|D)b^dY~O0jRA z@Lp7{YnH(spm$=B^XA6a(CgBxus-X~<=E>SQsO>#zfV@haN+rDm3U&$TY9Y7_efc~qI24sjg5fmNWMl;*IGP{lSSRMBA2C;xI1}iT)}Oo zYGw>uOk2Qre->vm-iji_T9Rh=xR=y)NWu+@)?6{khQN=YVV35Po)E7RJT4VPMlgu) z?~Yx_?~)?dTrS0glRQQE0GfW_b1VDCoUOjQH^-%F^3P?tZx*x+(^|S$yJK%ki)Ne` z0xzqwq?qv;-!qqGy(^7I>xW)g?Cac&DU4LPx#&A&LOu+kS&xG8oVIdVISV~;_E@VN zUK0V&6OlO21b4gsU~dWQv*LcYEAMs)nof*7$a6dU0V$zRCwT3cEyG&>>Ag?0V(-X0 zh)uQlfM)b=FxR8rvzyNSlaKR_40S6nAtkoF+ZWW>MWPQcZE&dBu){jnirgch5q2%aY{<^k7ljHC=Q&`jn5~r=^<;6kbfe+sM7V81-km8MK29O z&mkpI*pSUz&x9{yWq_wp;@_ESu$L!mwc%um57$a%x=&PBO7;4Rewfr+_I)#o60m4p zyc<5I)|z4dYUTdmo%f=p&@tnauXK`#pG(TUq->#297btvu$HQo(jQObN$o$ho)X!wY;RXCCw-cXd39BNb7tC5GAW9vdf!j`Dc3zpt-N*~v0T|i@B;kd^c`-~^ywJ(y~=1ja^)9cUgJsmzVA_#>oKT-vFz8}6GBz5-1@19t$ zQ%M;(cu1L~CqaRU>M7YV*9EoxU8?*p#BX%ZYLHA*sfy~4$9szQNy-n)jh@ostAxV2 z)#C&QZ8dck#1fw!jS0!jtQPok8`?*$>G8_Dpor!Xf$+MC2wtFMale(9vDX!gmi(y0 z!MugEZP&sP!Aj4B;o!_2N&fC*GeeIOyUe_LV zW$e(K1wCl2OztP@m$VCx)*Vi$&pw0~KUQt;5yKVL?VjGga<$x#KfMkwlQsD-L`2 z`7n%DTd-d*h#eNPnMG-BRPg%coj)m`Qun^I#m+`gj^aM9rZgvkd@tG&0hQw7J5sr-o&x3REmiPuq_A*iM3%#elKDd&U zP<`^E_3e1#I^fxmfP~L*u*%2tt9d3Yqjc9Dk!%MF)C(-;pkVAGKEIG&jUx3M{sMlq z^-le2OT@{E^2d5Y@PouCqS_5k_f+F<9-LfXbDK1oFLbvYSf|b|*ssmLz^Kanvg)0W zVjF&7m7`1j&u5rK5^1og7+4Cue;Il3iL+-PXHc+jExXz=-XTA_Hxb{KebAShEr6!)) z=kBSrWU2R$__;>;IQ|au5Y?<&VM@8lJzn0>RpSRryKdd&9LFf(Y|U)J7!%n zz?Hq%w1G^-D6=QErvBU#eP2=DNc>S3Uz*xhS$^T`(l0mUGBQ=(RD|^(kHH1s zR7(!%eGQB9!Cn-P+7Xqh(1u$s`m*m^mv7xHY(#Oe6zqSyM2-Yos0m9MjWGgp#_Lr; z+9hPN+quEmZwZ-k9EUcJla5~F=bklJ@mwhgeMD=H`Nnge_n5JZ|4Y*3Jm%r zYH^&kMBkmA6*|Mk#R@MtCO2+r3VI4_?)|-GMMgQas)j|%FZ6f)i#$;xp{f_B*BDMj_&P( z+-Z&%Y{$IQ-fj|mq?0_cuinQk&YJ9G5^$1RS7$*{_10DHu{}YWcwJp;y!`}*0gu~H z2W4<`T#7lOw|66IT|!RY`N>K9bpGr9>l61975Ej}B9Ylij<49Kw4Pl0tm&V91K0iN zMInM&byd0jbTj)k^`Gar%C7i5r|fFE3uSXhyGRuaKP(v{?^flKA}|bG!7bJqaSM{i zk=^LRNAnqka@e<4Q1F}xoIAs0ds=0ZO8n;1o)4t>({1$J46MdJRpA7m!WSXNr-Hnv z>1pmO$s2A_B>h`t!aYRUqWD-RswPQk#rLSnuC>IMd(%_Cq@rjW^QNbGN!?)8UoK)d zX_%)}H(pZ1QjFqv;J@IgK$aeLJz#- zax4P-hm4}v`N-|6*-Q4NTlB_zCsDm*i(c+unQqKIfIRwHR%Jg{xdY)KT7mPTr<371 zL;$pxbeVR<&ORr{!MVzw@Kbc2M=xjb^~vL5?w@x0fCf_sgJ~G<+D} zdF=I+_@+k{$(~R9>t|azy2EX}XCK0bgf#1?j&-|Dsxn^Gzfed>j5rr`pxlO3#y@}y zVg03@E@n@qw6&LQs{Ni%1U=`kG2vMEJP2>;HQ8sJL4`Hpvl;hEWeloHI_`yd{P}4J zsej#q6%)5LtN)U{W43M}?To#Avff>Uemyx?S|L{Z%0%O;C=Wl*NcT0%_gSsEcd!@F zO|0}i)8G4&(9N4~LDFgi^qS@owrvvCU~O=NxF%Zu4xqgZvOt|Q+v5Mz8XZ=^OgKhWr0gW9j`aY z76#G_^OmK*FmU1#4lv{QXUppfpt(8L4o`b5ba|hrqA_J)hbCRmTc3M)q|$NWYQ*4E z{GL2{Hx(Z@GKIFMwpSKbrO)rBxd^zR#pU(*1wj)vKO$1+V>P4|TxpEPW%C(aPCZr9 zJuuH@t8Q?FCoH|l8t0f2nXPB%aC7D9bI~LGm+Tz&J_~5BN_n~_vcy-I{L;y>x~krp z@9@(Djr;L}5W&5-J=cnS>+{5zlkBYQWgWdrvd!?@AE8jim3S-C2=?;6)l1baX&!Z2 z1(r>%*X@xRFAfAlQJ-+Z9<`HJ4&r>c93CmpZ;DLy4OYsB66($rE!y@~Ke)^qJCv3r z`%IV`SRoDG2l*RT`=x@}WKJ2jUft4ryvAY_DrB;Vuu%9;R!*FKBh$qDRiW0i-5WDO zcHR4FJQ*}0Xl#0wq|=6CMXnqzP7{RtC2~@3&jvF;jobYzrA89yeN?tj`ZxM<55K&Y zg3M4XLcA__doRFIXbzrYxK$QM=hl9DhT<*ru;U}CMwgNLb zhv{i(0v5pE^*c(h3pd+}ad0V}3x6D69Em#pn<#TQYhigt+2goExrzRO7AI$oQKpr( zL&g`%!-vOM7*vY_6G;4do&EJ*|Il!+Uw6xT`4v)L}#|mO||{+1baW|1ok?mWZp?dJl;C+#(WvUYgSP6k` zho}(wbV%He36){3Uy=BJdXx9vttyi1G>^KMia1hY*w#wxnj()(y&qT0J=qk^(Z%-g z?D$2cunebnw&2Ay_&(GWsf@QFdSbcOg4h&nw8RVI-=Kv2dn09!cikk^Nj12U-gSxg zO3v#jCZYJgMB($Lw9H5U1;{^j8G>3m-`PjzP7<%r(?MLw*mqjH|tkiWP3%TCxg z>SfPi%b$+)$(Zdv+!N3H5v)iV>TFB+_P(Vbv34*UENome92$RAVWgpd#`Vh#GXb(X zzzlVjJrV$TY4NcVv-+ z^$P>qF{WncDmu!h?)K-{ zpGT)uKXIcFYJDW=BTjl2FRUr`1@d>5j$!YL-jV=DG8v>c8msZ6xNNffv1~G2Zz6FiFJ1f~A&JH> z7cUsr2RU*&A>V#k5Y5mKk7(71ySK-sW{IwTc2#SEa6+`lMHK82dR3dhufgs~JutbB z|7o>a?7S#`keo!yTJQ1~db?+=iB*D}c!s#ZmDG?#xhb6#|>Ck2w2yO80W?MXb2RkLBA$hP%4PfjCWSq`cK zs3Jy(`b+s_(J@VzF5^ytw;_Wo4ytK5Ii5p17}7fMpyhy z6%mJ&u;WuRl!P%MDF8uevKAltI-vfU}BHaU0M1$M8^;@e}5 zty!j1#1Ok~=eAYy_eq(i6!duD zK3`-_)UEYt(81fSF16Bv!zZd1xo~h-0D*9Fzo^}Itu8GrTTQ-Zy>VQ;e0txPf&q%m zyvA86Fg^`taFv#tSh-GSPHFo^J^s&*KsKOX39r};3#E!Q`z=$OCy}3X z6@ND5HJj6f1SK=p{b*s1wDKkDp7Y4*&Z<*w-EWZBweU zh$;f_+XL&!3pA96P`6XPuH6~FVDf3gCQJ&w_mvgb3n}&v1loV~-Yjl;(ai-1RkJzU z*^H9Po+ZN=hx&W+E-EJ|Gwr(FMP}-1G%W0IOu6t@J2ZN#sSd5y=oqDI5EEjlw3j*v zB{}vsg$QKP!nZ*TPOH5E`{Yvc%c@TzJDJ^Uw=R|NRL6yAHKt8tPq8hrg}!xwIHF~G zgk{Ta%e3Ch1T&+1h83o!B=uDrPo{{x>aZJ;&ZTbAD9zTWY1W9_V!>~S`jl2@ow;LT z3=rveCG%xyM-SH~qViFM3TGY{zqmp+q=AY>eZB7Dx^XwQzEic&x5kXG*!`U?t@0w7 zvg;%(7+LOk6;GSTeAv|2uRP7-xFAYoJTEZNK@!>9QdfVAWJQv85jx1(!SD6Eh+#dW z{V6+dZ`xCfu8G1uS1a$nFLhr$m)S=7%1R?E(~b1A^BrwLH{t8PHuS{|Wei$_c8~G$ zaNj{?$V8OV^DDJy(*<4U@)j&qI0xzYpAt(UJ&m!8Ne%GtmFADnW+)-J+OMS87YY~l zo@_682#!jW>Qg6*{3MlbF~@v%MYugNOP5TaLWydLjp&el9PpF& z4m~z!+Q5TE`aGqx?s;-G{$Y}J@3WNn#q&a&^pQ&M7S2UX>eo05&oHykuOm_ZFV%@r z24U6^6Yr6`#x0MTO+oKp>h{^aw3~iJy=Pp?#>FZNg}zZc~^Y;$f1y z9aGW}E-TaAS$5o&09OSauN7KeCZ(f@+flQ(dl^BpCsamB%o7b<*>JtZAHE(&Bhbnh&+=Oi5yBhD z<xW(xqck<{*GE&a_gb&{L1=>Yy~ZG-f}082U1hV%qS9_& z&xxMH(?2hFBh)_BFk$b8d<5;iBcB;mX>cx;%Nc?EjlIaZ zA;vOx0cR)%=h>bZ>iFKMMbZ9e3qkkdXTS7@tCe-=9uOK_$Z3)`8+mq1(c+xVOTH6( za3YQcg+e$8q%=7W3puxO0LbH>B=?t3_a{V<6YS+b3L%gPuzAFC8ij*JfU_ro1wzVW zQT;#3YV`><7~4h#9nA^q!EDsRfeT8`Iu-KRAD@IkxOL<`WdX$k!qbD2Ee zaWY*}%p`DO3LvacR4a@q&oFEJPZQLXw8?}!IA+;uP9~(~Clx)l_ zd3sqPAv9qB+d@OP87llN4$q5z|>JYDWjBahZT5wLPghefMEzAlyq7Q|h zXcV#${90e^?!_qDc_Go}e9Zmw4N<~Z2f1eNsG&vMIkAx_ikD}hB|A?FJAY~hFP}S0 zDsPtUG@^`~nyAFVS(kza3;9b5S|a(HJ7*vQ+S#m=J+TP~MvtT$1dMfGOD>P zLzD-3ab;qi{df%aykhbzk|s#i3R>>If3r4ktn1lgjhK*w`b!D|oKjU?O5_E<-}M^! zOdmE#?}u1M8B`-pBaWXo&~16u6r?)O zE<@to#*aw9e2jPgmUn!t?0VWsiBZZ|TrICUJ|mC;S>p8JF6Cln;HDL4vk^}5Z%eT; z>djzJFj@f}x*;%tu3s0AId@$5l<88YE$a1~4kc|L#zK-F8~aZ6o>XwSF>ceNt>HBo zaW8>_fFT^%heTzo4)a3Bvd9$@{d;08Hb*WUn&CsGpkxyI)g~A;b)P5D&I`<|@@dqL z_9urZ)R3N|FK_QVtzoMuV^4CF1%bRrD1Tm2rl|T+jOxQ1^3s-TYcaA3g-h)!bYupH z9_>d?7NY3VNe#GktPFx5Tv+1d8RIUxebVFIrz1y4B76vtoxmYj>zIw>z1F2PasqQ? znAxQKeq6|S(5x}RWnCd8a5h(f`cPZ8-GxDU_ET7QAV;bbscJHGuKr^5V>shai2`4H z_9SIGAfl5MBbPhGX|=KgGYXx&Sjoc|fs2Xh-t>_Ee*f}1Pf=+nd?9tCfNak}Tl3zl zbK0IKGiij?_=%Q?6yImPHJPR+a3{oZr5v|+49jkbHY=c&xI?EhEbUq|MW=;>Bv?51 zVl&-UIq1|wtl$-erO11GAXL27bjFvJG9G#%T2->nS-tEL?XD{IcO?rF z@$XgS$L6WY+^?wA)YL$BGJB~+?+>YC7f%dPt~hW!8g^5cYaO^_Fe%PT0D2(f;>{Ao=--#d~Yb?ulh1bn*h1 z9!}zOq$J)L{~T?4U7ww`WQGMP&v@r9e)Ohl5Ss075?JRwp# zujo@TIQd|*SAxLLl7(KUIU#L9aB|Py=E;q)8n#iA>7oV+xTgHz< zv~Zx2zgj1|?6Pa~fne_ahB6`73^nc^+W%%>cJ@>GN7Tp>PSSF!1J)<&S;lO7%TmCa z($3xRP?R5Mnu{vsm$H^vZn{f~oaG+4=?@Q-(P+2eK*r7B3{vOZ^G*pzkYskRh2Zr6sXE^iqihR}tjM4291R&K*0IoWc+i z7F1I=&VGB6eI}I<++6h+S?;ebx#b<;B;ts7m++BrIa(%kVb;KCjXRwt|JZ3oG30)ViJ_}$XMm#@jdy+lgT&*q=fcI+cZCw%kdjkIrE8*TY}mjKAw9regnQc z1_nxnVvUcUJsN6XG>pyf+lz;);wThg?PsFNx%oDF6t}G;r29o$PELkwoa$q_tjk6T zLlbd4T!z1msERZoSJ3PhPs%s0e|r94;^jS@<1M%>&3DN@G*!h{o|IZGU6^=LVUH$# zbf&HAs>N|+^ut#7rl!?~Dc;v+1>lG%HI=7hbg(>V^1(Fkgnk%2_qYK5M|XA_v6<*N zn|?~Vp?G==q$Qt`pq+E8Rz|micU7TD7u~-VRArvav}x>aai*M!Jo_X*Zx7vAp^eyN@XgFdi~D0qlTH~T zqFeI940BanBPvy+C1Zo0jMr>pGqaFS=Ie+3euVrpm8H}AKC#o1r3|Vn>nYiM7TxzJ z9T9#OH3dfAiM_UIuktFN9Sh-1TOqxZRuUM~rfC@Mm~?QUhVs4I?G%kQMCBfNM|Ols zE@hUMOmOG87eVbNZnDcp2M4)&pvm;oP5ud$Ae!(|rTmOr_?i-~O!d|C+I!=4)6JiE zdu3*h;J^B94BRlW3CJ~Tq7w59c~mKpwKT`XdgLIF)tKrnephgj#pLM4`!`?Ch+gMF zeU9NAZf`MB?r2%}-5$rKaEU`1PixhwRUtJ3jacE#75rc+`A=4eF&8 z8{S*jB6AH~DN%5;5e$-?QMWj+&`w^aN@>#;DH5Q6Oei}$?Ce0^=NZI-cU{5cAv()# zqmHpDCyLbfUAO7%n_OliQW_n(4o?ZSsSVm5rsc55NX9==X0^?q7A-PqJw9RU=rEA) zXpj`=7Mu;afW?ROF^#0wJezr1Lzx>C7NaZ$;#v_mCRtUrRb#B%L z(1^ykQpHjrdrC2EDt8La!MkTPF8TAIEscniZkzUG#~G2-poQfaSyt|bu{26>u|5r9 zcV)s&RMz*nZa3o`;u814+-7X0cVXw`!Id55DCs4E`NAZ5mqmspxr_0&!~_T@o!I>$ z*lqXLSji1*@3XdDLwQ|0!iF!lPs*2W+yw>JY|=blw6=Z z`+P*D=~ET!ixmfsf#GuxQM-u>i9n}uCzcDrE2OG?VYXOwjW=D-T5+qAXI)9x%nGBO zr>TBNx7=e9^n|ZfZH=+Yx6155v?I9l>B8} z{V1@hqpn7IQ*i1igD#5#8S87Ql8wDfUFIO9q~gDQh6b z^YAs<5>jvYYa0_7i`6eZj!7-gD13N*xXRT>_vxlh0Q3o|NEEmEdxDBoeW=VZ zQ211E#L>3|xJa5Hn(3Nn_$v}YK82^AJz94be)sQ_ z?i?Z(2%@tu?sB>ZZ5b*u!nET&Is&+hqf%h#cK zL#)K-%v3US0K^ctAcN<0B|x5-Np!q~nl1H|T*c64QdiYQin+;HJ-SOEu+1F%Q@zg7lbQZh%{@9m3q~xYzDMFzB}f(%`tA`TqU%nH zrj=|k$C&y@bXbBfgv!zU?58X_u#Q+rQY5PN%@|p#uDR?l|JL7XkvxQ*%=ENc2g&|& z<`Z$x&nHP^d5w>sbw-~OwmSQ%tQrWJr)(eus>dI#xp0jY604qLNeJ@1|LG(;^~6Ia zT?9q_=G9s`tmWJ^A2R=KIXbK&Ly&&!m6)+{&H>ygqhu3WQm(t}KQa3n4 zlE%iZ!Cy((MCHyRkn$|#1Ymmp_Y;(6BO2Q|FoQ;8)KCHZ*%qX#tgdul!8qXm1MK`m ARR910 literal 0 HcmV?d00001 diff --git a/res/icons/setup.svg b/res/icons/setup.svg new file mode 100644 index 00000000000..0d500c4f787 --- /dev/null +++ b/res/icons/setup.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/res/setup/.editorconfig b/res/setup/.editorconfig new file mode 100644 index 00000000000..12383cacede --- /dev/null +++ b/res/setup/.editorconfig @@ -0,0 +1,9 @@ +# indicate this is the root of the project +root = true + +[*] +indent_style = tab +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true +charset = utf-8 diff --git a/res/setup/readme.txt b/res/setup/readme.txt new file mode 100644 index 00000000000..74622688f51 --- /dev/null +++ b/res/setup/readme.txt @@ -0,0 +1,29 @@ +Rufus: The Reliable USB Formatting Utility - Windows 11 setup.exe wrapper + +# Description + +This small executable aims at solving the issue of Windows 11 24H2 having made the +bypass requirements for in-place upgrades more difficult to enact. + +Basically, per https://github.com/pbatard/rufus/issues/2568#issuecomment-2387934171, +and if the user chose to apply the hardware requirement bypasses in Rufus, you want +to apply a set of registry key creation and deletion *before* setup.exe is run. + +While we could obviously provide a simple batch file to accomplish this, the fact +that the registry commands require elevation, combined with expectations of just +being able double click setup.exe to upgrade makes us want to accomplish this in +a more user-friendly manner. + +Our solution then is to have Rufus rename the original 'setup.exe' to 'setup.dll' +insert a small 'setup.exe' that'll perform elevation, add the registry key, and +launch the original setup, which is exactly what this project does. + +Now, obviously, the fact that we "inject" a setup executable may leave people +uncomfortable about the possibility that we might use this as a malware vector, +which is also why we make sure that the one we sign and embed in Rufus does get +built using GitHub Actions and can be validated to not have been tampered through +SHA-256 validation (Since we produce SHA-256 hashes during the build process per: +https://github.com/pbatard/rufus/blob/master/.github/workflows/setup.yml). + +Also note that, since these are the only platforms Windows 11 supports, we only +build for x64 and ARM64. diff --git a/res/setup/resource.h b/res/setup/resource.h new file mode 100644 index 00000000000..75aac6a490a --- /dev/null +++ b/res/setup/resource.h @@ -0,0 +1,16 @@ +//{{NO_DEPENDENCIES}} +// Microsoft Visual C++ generated include file. +// Used by setup.rc +// +#define IDI_ICON 101 + +// Next default values for new objects +// +#ifdef APSTUDIO_INVOKED +#ifndef APSTUDIO_READONLY_SYMBOLS +#define _APS_NEXT_RESOURCE_VALUE 102 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1000 +#define _APS_NEXT_SYMED_VALUE 101 +#endif +#endif diff --git a/res/setup/setup.c b/res/setup/setup.c new file mode 100644 index 00000000000..1ad736b581c --- /dev/null +++ b/res/setup/setup.c @@ -0,0 +1,125 @@ +/* + * Setup - Wrapper around Microsoft's setup.exe that adds registry + * bypasses for in-place Windows 11 upgrade. + * + * Copyright © 2024 Pete Batard + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#define WIN32_LEAN_AND_MEAN +#include +#include +#include + +static BOOL RegDeleteNodeRecurse(HKEY hKeyRoot, CHAR* lpSubKey) +{ + CHAR* lpEnd; + LONG lResult; + DWORD dwSize; + CHAR szName[MAX_PATH]; + HKEY hKey; + FILETIME ftWrite; + + // First, see if we can delete the key without having to recurse. + if (RegDeleteKeyA(hKeyRoot, lpSubKey) == ERROR_SUCCESS) + return TRUE; + + lResult = RegOpenKeyExA(hKeyRoot, lpSubKey, 0, KEY_READ, &hKey); + if (lResult != ERROR_SUCCESS) + return (lResult == ERROR_FILE_NOT_FOUND); + + // Check for an ending slash and add one if it is missing. + lpEnd = lpSubKey + strlen(lpSubKey); + if (*(lpEnd - 1) != '\\') { + *lpEnd++ = '\\'; + *lpEnd = '\0'; + } + + // Enumerate the keys + dwSize = MAX_PATH; + if (RegEnumKeyExA(hKey, 0, szName, &dwSize, NULL, NULL, NULL, &ftWrite) == ERROR_SUCCESS) { + do { + *lpEnd = '\0'; + strcat_s(lpSubKey, MAX_PATH, szName); + if (!RegDeleteNodeRecurse(hKeyRoot, lpSubKey)) + break; + dwSize = MAX_PATH; + lResult = RegEnumKeyExA(hKey, 0, szName, &dwSize, NULL, NULL, NULL, &ftWrite); + } while (lResult == ERROR_SUCCESS); + } + + *--lpEnd = '\0'; + RegCloseKey(hKey); + + // Try again to delete the key. + return (RegDeleteKeyA(hKeyRoot, lpSubKey) == ERROR_SUCCESS); +} + +static BOOL RegDeleteNode(HKEY hKeyRoot, CHAR* lpSubKey) +{ + CHAR szDelKey[MAX_PATH]; + + strcpy_s(szDelKey, MAX_PATH, lpSubKey); + return RegDeleteNodeRecurse(hKeyRoot, szDelKey); +} + +static BOOL RegWriteKey(HKEY hKeyRoot, CHAR* lpKeyParent, CHAR* lpKeyName, DWORD dwType, LPBYTE lpData, DWORD dwDataSize) +{ + BOOL r = FALSE; + HKEY hRoot = NULL, hApp = NULL; + DWORD dwDisp; + HKEY hKey; + + if (RegCreateKeyExA(hKeyRoot, lpKeyParent, 0, NULL, 0, KEY_SET_VALUE | KEY_QUERY_VALUE, NULL, &hKey, &dwDisp) != ERROR_SUCCESS) + return FALSE; + + r = (RegSetValueExA(hKey, lpKeyName, 0, dwType, lpData, dwDataSize) == ERROR_SUCCESS); + RegCloseKey(hKey); + + return r; +} + +int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdShow) +{ + CHAR lpBypasses[] = "SQ_SecureBootCapable=TRUE\0SQ_SecureBootEnabled=TRUE\0SQ_TpmVersion=2\0SQ_RamMB=8192\0"; + DWORD dwUpgrade = 1, dwAttrib; + STARTUPINFOA si = { 0 }; + PROCESS_INFORMATION pi = { 0 }; + SECURITY_ATTRIBUTES sa = { sizeof(SECURITY_ATTRIBUTES), NULL, TRUE }; + + // Make sure we have 'setup.dll' in the same directory + dwAttrib = GetFileAttributesA("setup.dll"); + if (dwAttrib == INVALID_FILE_ATTRIBUTES || dwAttrib & FILE_ATTRIBUTE_DIRECTORY) + MessageBoxA(NULL, "ERROR: 'setup.dll' was not found", "Windows setup error", MB_OK | MB_ICONWARNING); + + // Apply the registry bypasses to enable Windows 11 24H2 in-place upgrade + RegDeleteNode(HKEY_LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\AppCompatFlags\\CompatMarkers"); + RegDeleteNode(HKEY_LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\AppCompatFlags\\Shared"); + RegDeleteNode(HKEY_LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\AppCompatFlags\\TargetVersionUpgradeExperienceIndicators"); + RegWriteKey(HKEY_LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\AppCompatFlags\\HwReqChk", + "HwReqChkVars", REG_MULTI_SZ, lpBypasses, sizeof(lpBypasses)); + RegWriteKey(HKEY_LOCAL_MACHINE, "SYSTEM\\Setup\\MoSetup", "AllowUpgradesWithUnsupportedTPMOrCPU", + REG_DWORD, (LPBYTE)&dwUpgrade, sizeof(dwUpgrade)); + + // Launch the original 'setup.exe' (that was renamed to 'setup.dll') + si.cb = sizeof(si); + si.dwFlags = STARTF_USESHOWWINDOW; + si.wShowWindow = SW_SHOWNORMAL; + CreateProcessA("setup.dll", NULL, NULL, NULL, TRUE, NORMAL_PRIORITY_CLASS, NULL, NULL, &si, &pi); + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); + + return GetLastError(); +} diff --git a/res/setup/setup.rc b/res/setup/setup.rc new file mode 100644 index 00000000000..0be07642278 --- /dev/null +++ b/res/setup/setup.rc @@ -0,0 +1,116 @@ +// Microsoft Visual C++ generated resource script. +// +#include "resource.h" + +#define APSTUDIO_READONLY_SYMBOLS +///////////////////////////////////////////////////////////////////////////// +// +// Generated from the TEXTINCLUDE 2 resource. +// +#ifndef _USING_V110_SDK71_ +#define _USING_V110_SDK71_ +#endif +#include +#ifndef IDC_STATIC +#define IDC_STATIC -1 +#endif + +///////////////////////////////////////////////////////////////////////////// +#undef APSTUDIO_READONLY_SYMBOLS + +///////////////////////////////////////////////////////////////////////////// +// English (Neutral) resources + +#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_NEU) +LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL +#pragma code_page(1252) + +#ifdef APSTUDIO_INVOKED +///////////////////////////////////////////////////////////////////////////// +// +// TEXTINCLUDE +// + +1 TEXTINCLUDE +BEGIN + "resource.h\0" +END + +2 TEXTINCLUDE +BEGIN + "\0" +END + +3 TEXTINCLUDE +BEGIN + "\r\n" + "\0" +END + +#endif // APSTUDIO_INVOKED + + +///////////////////////////////////////////////////////////////////////////// +// +// Icon +// + +// Icon with lowest ID value placed first to ensure application icon +// remains consistent on all systems. +IDI_ICON ICON "../icons/setup.ico" + + +///////////////////////////////////////////////////////////////////////////// +// +// Version +// + +VS_VERSION_INFO VERSIONINFO + FILEVERSION 1,0,0,1 + PRODUCTVERSION 1,0,0,1 + FILEFLAGSMASK 0x3fL +#ifdef _DEBUG + FILEFLAGS 0x1L +#else + FILEFLAGS 0x0L +#endif + FILEOS 0x40004L + FILETYPE 0x1L + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "000004b0" + BEGIN + VALUE "CompanyName", "Akeo Consulting" + VALUE "FileDescription", "Windows Setup Wrapper" + VALUE "FileVersion", "1.0" + VALUE "InternalName", "Setup" + VALUE "LegalCopyright", " 2024 Pete Batard (GPL v3)" + VALUE "LegalTrademarks", "https://rufus.ie/setup" + VALUE "OriginalFilename", "setup.exe" + VALUE "ProductName", "Setup" + VALUE "ProductVersion", "1.0" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0, 1200 + END +END + +#endif // English (Neutral) resources +///////////////////////////////////////////////////////////////////////////// + + + +#ifndef APSTUDIO_INVOKED +///////////////////////////////////////////////////////////////////////////// +// +// Generated from the TEXTINCLUDE 3 resource. +// + + +///////////////////////////////////////////////////////////////////////////// +#endif // not APSTUDIO_INVOKED + diff --git a/res/setup/setup.sln b/res/setup/setup.sln new file mode 100644 index 00000000000..dc034893b6e --- /dev/null +++ b/res/setup/setup.sln @@ -0,0 +1,37 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29926.136 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "setup", "setup.vcxproj", "{6C2BED99-5A0A-42A2-AEBE-66717FA92232}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|arm64 = Debug|arm64 + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|arm64 = Release|arm64 + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {6C2BED99-5A0A-42A2-AEBE-66717FA92232}.Debug|arm64.ActiveCfg = Debug|ARM64 + {6C2BED99-5A0A-42A2-AEBE-66717FA92232}.Debug|arm64.Build.0 = Debug|ARM64 + {6C2BED99-5A0A-42A2-AEBE-66717FA92232}.Debug|x64.ActiveCfg = Debug|x64 + {6C2BED99-5A0A-42A2-AEBE-66717FA92232}.Debug|x64.Build.0 = Debug|x64 + {6C2BED99-5A0A-42A2-AEBE-66717FA92232}.Debug|x86.ActiveCfg = Debug|Win32 + {6C2BED99-5A0A-42A2-AEBE-66717FA92232}.Debug|x86.Build.0 = Debug|Win32 + {6C2BED99-5A0A-42A2-AEBE-66717FA92232}.Release|arm64.ActiveCfg = Release|ARM64 + {6C2BED99-5A0A-42A2-AEBE-66717FA92232}.Release|arm64.Build.0 = Release|ARM64 + {6C2BED99-5A0A-42A2-AEBE-66717FA92232}.Release|x64.ActiveCfg = Release|x64 + {6C2BED99-5A0A-42A2-AEBE-66717FA92232}.Release|x64.Build.0 = Release|x64 + {6C2BED99-5A0A-42A2-AEBE-66717FA92232}.Release|x86.ActiveCfg = Release|Win32 + {6C2BED99-5A0A-42A2-AEBE-66717FA92232}.Release|x86.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {7994FC0C-9D7C-4AD7-855B-C3525FD8709A} + EndGlobalSection +EndGlobal diff --git a/res/setup/setup.vcxproj b/res/setup/setup.vcxproj new file mode 100644 index 00000000000..b4043bc7a0e --- /dev/null +++ b/res/setup/setup.vcxproj @@ -0,0 +1,272 @@ + + + + + Debug + ARM64 + + + Debug + Win32 + + + Release + ARM64 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 16.0 + {6C2BED99-5A0A-42A2-AEBE-66717FA92232} + Win32Proj + baseconsole + + + + Application + true + Unicode + v143 + + + Application + false + true + Unicode + v143 + + + Application + true + Unicode + v143 + + + Application + true + Unicode + v143 + + + Application + false + true + Unicode + v143 + + + Application + false + true + Unicode + v143 + + + + + + + + + + + + + + + + + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + $(SolutionDir)arm64\$(Configuration)\ + $(SolutionDir)arm64\$(Configuration)\$(ProjectName)\ + $(SolutionDir)arm64\$(Configuration)\ + $(SolutionDir)arm64\$(Configuration)\$(ProjectName)\ + $(SolutionDir)x86\$(Configuration)\ + $(SolutionDir)x86\$(Configuration)\$(ProjectName)\ + $(SolutionDir)x86\$(Configuration)\ + $(SolutionDir)x86\$(Configuration)\$(ProjectName)\ + $(SolutionDir)x64\$(Configuration)\ + $(SolutionDir)x64\$(Configuration)\$(ProjectName)\ + $(SolutionDir)x64\$(Configuration)\ + $(SolutionDir)x64\$(Configuration)\$(ProjectName)\ + false + false + false + false + false + false + + + true + + + true + + + true + + + false + + + false + + + false + + + + /DAPP_VERSION=$(AppVersion) %(AdditionalOptions) + + + + + /DAPP_FILE_VERSION=$(AppFileVersion) %(AdditionalOptions) + + + + + /DAPP_COMMENTS="$(AppComments)" %(AdditionalOptions) + + + + + Level3 + WIN32;_DEBUG;%(PreprocessorDefinitions) + false + CompileAsC + MultiThreadedDebug + + + + + Windows + true + RequireAdministrator + + + + + Level3 + _DEBUG;%(PreprocessorDefinitions) + false + CompileAsC + MultiThreadedDebug + + + + + Windows + true + RequireAdministrator + + + + + Level3 + _DEBUG;%(PreprocessorDefinitions) + false + CompileAsC + MultiThreadedDebug + + + + + Windows + true + RequireAdministrator + + + + + Level3 + WIN32;NDEBUG;%(PreprocessorDefinitions) + false + CompileAsC + MultiThreaded + + + MinSpace + Size + + + Windows + true + true + true + RequireAdministrator + + + + + Level3 + NDEBUG;%(PreprocessorDefinitions) + false + CompileAsC + MultiThreaded + + + MinSpace + Size + + + Windows + true + true + true + RequireAdministrator + + + + + Level3 + NDEBUG;%(PreprocessorDefinitions) + false + CompileAsC + MultiThreaded + + + MinSpace + Size + + + Windows + true + true + true + RequireAdministrator + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/res/setup/setup.vcxproj.filters b/res/setup/setup.vcxproj.filters new file mode 100644 index 00000000000..31d4927c846 --- /dev/null +++ b/res/setup/setup.vcxproj.filters @@ -0,0 +1,27 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;c++;def;odl;idl;hpj;bat;asm;asmx + + + {e9f75410-9e5d-4ea8-b75a-0c45bb23e331} + + + + + Source Files + + + + + Resources + + + + + Source Files + + + \ No newline at end of file diff --git a/src/rufus.rc b/src/rufus.rc index ffba8ce100a..9c702576631 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2198" +CAPTION "Rufus 4.6.2199" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -397,8 +397,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2198,0 - PRODUCTVERSION 4,6,2198,0 + FILEVERSION 4,6,2199,0 + PRODUCTVERSION 4,6,2199,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -416,13 +416,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2198" + VALUE "FileVersion", "4.6.2199" VALUE "InternalName", "Rufus" - VALUE "LegalCopyright", "� 2011-2024 Pete Batard (GPL v3)" + VALUE "LegalCopyright", " 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2198" + VALUE "ProductVersion", "4.6.2199" END END BLOCK "VarFileInfo" From c800448c620a3767565e038e29739a9dccfe5e71 Mon Sep 17 00:00:00 2001 From: Pete Batard Date: Sun, 6 Oct 2024 13:39:53 +0100 Subject: [PATCH 18/30] [wue] add setup wrapper to add bypasses for in-place upgrades of Windows 11 24H2 * Per https://forums.mydigitallife.net/threads/win-11-boot-and-upgrade-fix-kit-v5-0-released.83724/ Windows 11 24H2 requires new registry bypasses to be applied to perform in-place upgrade on non officially supported platforms, and those need to be enacted before running setup.exe. * In order to streamline this, and because those registry bypasses require elevation, we rename setup.exe to setup.dll and add our own setup.exe wrapper to set the registry and then call the original setup.exe (through setup.dll). * See https://github.com/pbatard/rufus/issues/2568 * Also fix some MinGW build warnings. * Also fix the annoyance of TortoiseGit/Notepad++ altering the copyright symbol of rufus.rc. --- _pre-commit.sh | 1 + res/setup/readme.txt | 30 ++++++++++++++++++++++-------- res/setup/setup.c | 5 +++-- res/setup/setup_arm64.exe | Bin 0 -> 144688 bytes res/setup/setup_x64.exe | Bin 0 -> 152880 bytes src/hash.c | 2 +- src/parser.c | 33 ++++++++++++++++++++++++--------- src/resource.h | 2 ++ src/rufus.c | 10 ++++------ src/rufus.h | 1 + src/rufus.rc | 14 +++++++++----- src/wue.c | 30 +++++++++++++++++++++++++++++- 12 files changed, 96 insertions(+), 32 deletions(-) create mode 100644 res/setup/setup_arm64.exe create mode 100644 res/setup/setup_x64.exe diff --git a/_pre-commit.sh b/_pre-commit.sh index 12469c41ef9..3a2b65bf763 100755 --- a/_pre-commit.sh +++ b/_pre-commit.sh @@ -32,6 +32,7 @@ s/^\([ \t]*\)*\(FILE\|PRODUCT\)VERSION\([ \t]*\)\([0-9]*\),\([0-9]*\),[0-9]*,\(. s/^\([ \t]*\)VALUE\([ \t]*\)"\(File\|Product\)Version",\([ \t]*\)"\(.*\)\..*"[ \t]*/\1VALUE\2"\3Version",\4"\5.@@BUILD@@"/ s/^\(.*\)"Rufus \(.*\)\..*"\(.*\)/\1"Rufus \2.@@BUILD@@"\3/ s/^\([ \t]*\)Version="\([0-9]*\)\.\([0-9]*\)\.[0-9]*\.\([0-9]*\)"\(.*\)/\1Version="\2.\3.@@BUILD@@.\4"\5/ +s/\xef\xbf\xbd/\xa9/ _EOF # First run sed to substitute our variable in the sed command file diff --git a/res/setup/readme.txt b/res/setup/readme.txt index 74622688f51..005b521f606 100644 --- a/res/setup/readme.txt +++ b/res/setup/readme.txt @@ -1,4 +1,4 @@ -Rufus: The Reliable USB Formatting Utility - Windows 11 setup.exe wrapper +Rufus: The Reliable USB Formatting Utility - Windows 11 Setup wrapper # Description @@ -18,12 +18,26 @@ Our solution then is to have Rufus rename the original 'setup.exe' to 'setup.dll insert a small 'setup.exe' that'll perform elevation, add the registry key, and launch the original setup, which is exactly what this project does. -Now, obviously, the fact that we "inject" a setup executable may leave people -uncomfortable about the possibility that we might use this as a malware vector, -which is also why we make sure that the one we sign and embed in Rufus does get -built using GitHub Actions and can be validated to not have been tampered through -SHA-256 validation (Since we produce SHA-256 hashes during the build process per: +Oh and it should be noted that, the issues you might see with Setup not restarting +in the foreground after it updates, or not being able to launch at all for a while +if you happen to cancel before starting the installation, have *NOTHING* to do with +using this setup wrapper, but come from Microsoft themselves. You can validate that +these issues exist even when running setup.exe without the wrapper... + +# Security considerations + +Obviously, the fact that we "inject" a setup executable may leave people uncomfortable +about the possibility that we might use this as a malware vector, which is also why we +make sure that the one we sign and embed in Rufus does get built using GitHub Actions +and can be validated to not have been tampered through SHA-256 validation (Since we +produce SHA-256 hashes during the build process per: https://github.com/pbatard/rufus/blob/master/.github/workflows/setup.yml). -Also note that, since these are the only platforms Windows 11 supports, we only -build for x64 and ARM64. +Per the https://github.com/pbatard/rufus/actions/runs/11195726475 GitHub Actions +workflow run, the SHA-256 for the executables (before signature was applied) were: +* 4e99f49b456781c92d2010a626706557df69334c6fc71ac129625f41fa311dd8 *./setup_x64.exe +* a0d7dea2228415eb5afe34419a31eeda90f9b51338f47bc8a5ef591054277f4b *./setup_arm64.exe + +You will also find the VirusTotal reports for the current signed executable at: +* https://www.virustotal.com/gui/file/a74dbfc0e2a5b2e3fd4ad3f9fdaea0060c5d29a16151b9a570a9bd653db966a7/detection +* https://www.virustotal.com/gui/file/629fdda27e200ec98703a7606ca4e2d0e44c578341779e4d5c447d45fc860c69/detection diff --git a/res/setup/setup.c b/res/setup/setup.c index 1ad736b581c..641bfb66ec7 100644 --- a/res/setup/setup.c +++ b/res/setup/setup.c @@ -84,7 +84,7 @@ static BOOL RegWriteKey(HKEY hKeyRoot, CHAR* lpKeyParent, CHAR* lpKeyName, DWORD if (RegCreateKeyExA(hKeyRoot, lpKeyParent, 0, NULL, 0, KEY_SET_VALUE | KEY_QUERY_VALUE, NULL, &hKey, &dwDisp) != ERROR_SUCCESS) return FALSE; - + r = (RegSetValueExA(hKey, lpKeyName, 0, dwType, lpData, dwDataSize) == ERROR_SUCCESS); RegCloseKey(hKey); @@ -104,7 +104,8 @@ int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine if (dwAttrib == INVALID_FILE_ATTRIBUTES || dwAttrib & FILE_ATTRIBUTE_DIRECTORY) MessageBoxA(NULL, "ERROR: 'setup.dll' was not found", "Windows setup error", MB_OK | MB_ICONWARNING); - // Apply the registry bypasses to enable Windows 11 24H2 in-place upgrade + // Apply the registry bypasses to enable Windows 11 24H2 in-place upgrade. Credits to: + // https://forums.mydigitallife.net/threads/win-11-boot-and-upgrade-fix-kit-v5-0-released.83724/ RegDeleteNode(HKEY_LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\AppCompatFlags\\CompatMarkers"); RegDeleteNode(HKEY_LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\AppCompatFlags\\Shared"); RegDeleteNode(HKEY_LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\AppCompatFlags\\TargetVersionUpgradeExperienceIndicators"); diff --git a/res/setup/setup_arm64.exe b/res/setup/setup_arm64.exe new file mode 100644 index 0000000000000000000000000000000000000000..c7a918aaa8554b8b4099939548fa9a588b13095c GIT binary patch literal 144688 zcmd?Sdwf;Zng74`ISK5O074Q%Kx-24a&qu)K;osG5Vb)&Z3)=gIxUIR@q~=F5wIe* zoJegCrk#2KYl|~q0(7QHsxw$a%dyj%$V?~5w8N#^PQUXl0kkHhor0j|NHO2{XYaF< zoe*s2_4@w)`sMXH*?X_OF3)<_v!2^}*4lsZsV7X(7?Z_saL|}tT=f@q{=fe}+cIWS z@e7koN9bpzyQ~lYtaM4NenrXWANbt857gaPvaIg@`#-m`^)aZoS1V_rs{2n8vgRPKgmAN`#t;9ag$!-`PrYENq_BJtx3P*`u6qXCiOd4 zYf`UsHIq7>_gRw;bN%g4vd;T4*MGcW;-oj6>$~Ua`rDsuaIXJ&!+0mXrhZvWZH?~P z$8Is^v+d)|{0-K<-n-*wrpXOVDl@<t;X48#teTlbzeBlcM{8G2`@)KksTj#xqF$ zk1q#!Q`_Vf%9JIiTZwm3*-VOV%_9$3=3~0qK`WFtEtY8sakT^Hs>``X1LnqY#(b>S zGD~{~a#-e07gR4{#Py1m_dK+clz}(+CxFy<5r^-u#F*ME9{6nC$~t2n zdRQ-j@ewY+)BW&Y6ezE7vzc$rAU#gL30z}beScA7IY(JLNU z@xU^bZQ>`~Jj(S;W97T&uRljd;YheLHdiyU<56Q4DE0r(e>;phJ~?2LlLOIov6WkCtx-i^TA*E@mFFMzpCfKb!LC? zJMja{uG#+P(V%HNoNKj_mQ>yPd1X#6exdEHyQphfpe@}HY&*RotL@C1?6xyiR&vG1 zqHU}0tZe&2?Si(|v8uK;pTD`SE*woq&3xdyg}gUYri!u)sH3v&*Us_y?U>Kxo{ z?LV`|qKyD;1!*&jwzGj@TwAG$rVj^A`ee|wp9)&-Zv_MGZwG_zNBI3@aPX;~V1E1I zU;)ol+m8kd+ZWTXJLuOZ=+`Id*Io4M)AY;GFV(TjD|22ny}-l|Jezqwz2a8dyoI)J z28JqNSpcpo+knl%tA|tnHINmZw=K(*zfQedCq&09&{qG*x(DM01L+RTF2j`H!e`~<}t@&2TN6ZDKA2F?0-5;RL zf(fSP?@ge&#hAUq{oiMq{oZ{kWex|ez1s?+^A-#amX~Ik6VD6|F6kQ_oCY&Ep#0}e znm8dhCuENwmz@;^fWa!W-=(!^`oyxU_nsJU(#q%cN$-x2kER_OTz1vo;=q)CSius? zIXEaT3BJ-3oZQY#FHo5?YhHBd_*QUuW=)qvA6{L$I`sYK!qgMS%r)jP;?K%ed&iCG z2>fwrqKfC>A15bvfP^XEOC44wgo|IzF#DCLi84vV3XIaQT2!{u`A4 zNT$4H;(3Q0e7I1Js4MQ)wJ!sU1swYvINW;viF$6#)Z_JORQUnHk}2O!`R+{lcBQNQ zWuwZMnt08R&IQj^qx)t>#_Ah`>)>86O*YZgbHVt5DQ4H3j8#Ju^!KqZN7GXR(bUz; z&(AxrI#F}pcVBp5(3;Y)m2sS$XObWLvSBX7YXoCS=IZrtve|J!G;r0Mq10q(=wl1! zy)t>yf`gMS(=mA*_n%pEusFw@IDXxh&65Ll-&ejKtN7J@ga343@~3XAIPL43>Rglq zFI#bC|M6>|*eqJA%J9F*KVP}w>T%Ki^Wk;#1C#r=LbrnF%$k2@-dW7QK-)X#o3wc3 zVrXk{UDo~>aM{_>^r{?lXAg7zB)q^_^IxqsW{&!Sic#R?nG#LEHp?WJg-lU^^uzE6 z<=kCJ`{JH;pq%r@pDtbPsdln^KDqcO;NBvSE1$$Kde;FT{MTtG6ivSk|9y+OdMcR3 z7-UD@*zV%w9{LYHYJ%WIc&`nbJ3XA!OwAN;d?td=Y0>oS!TiW;@a?0)sS$Ua3Y{_X z`snr{nm$QC9K4cWeCW)Yms(wU^wL@V5Zt04(apcU2i;WJ`&*59BGh$F^6JXyvmM;W zXO<_!TRT@@3rr_=XFq#&W#zNI!NBV9!mf&w`6hK@_qb=Tn$mewZ*EqF1Iy*m@cj5q zlV39*Jzf=f_IOp0U)Hn77n?IqT5kNN&XzIW`950JGvhz>{%ZH0u}ao5 zW-g7oW4m@%@RgJ26%TlFCQ3Um($`llvuWxSvuPh!WP0RV+~*sUD&hSfjM-#gU>3Iq zP5Ecgp=^nvrin>-ST`o8~4BE$DA*B^8}JRs9WjEm*Q7Vy}GKtYV!TB z^US8_dHy567Wj^2-pTQm1B1XR{?$YKhj{1dZGzu}{YOrEyhdfb{QmbLVEtS2ET&C= z9>bsVIdd}?o|u`N>g(TT`dXc9%BKo9RilZVXY2Jzjo*CB?yp5xs6Ml_8yxRkcxFoXbLc)V-Vl5xJUF=10GGeb z5}yBvZ1=+#2Unt}HtG#0udh%)@78<+_tZH@b=b6ZL!O(K$fPxpCrw?d)8B^r<G^5gZeN zL+Lj8LMFZ;bg^j*g^Ys}o4nq$Gd~`^p~8dh0Xmql0G zR!*~BXOasiMpu`GLe04oL(Q$s^ud%(a=TXhD2O_}A5#gmYCLrt1dr|9$o?f~A;uIFg(^ z#yIb-8;9&-=eczF*^g&v^QlBu{6Gjh9=O|WW74Ncvw60ywcpqCX#A@_`!6(STs_v; z?r~EY@wa>2lny_O7b>sY{w`qrcv!Kp9y~mn4*Q2ef#{7X$*6Jb5r%h%=IOr3uo^5_}^EV zr}>Q4#&OZKHGOX{V|Z}F zn(CequX*scXbL&Ke5-AioWj3%6FlzMUy2^=%bU?0&So5|!u?0bnI&Q5f$ksG{gh~` zdYmZ_BQK=)m4Mr2k!X5mRk-;uHn8Tw>30X8>dYRvPCWAW@Uiv4Qa#BO(N@D@@~>fD z#F{J93G}$ystL*QSHYj9$B|YFU7+)|k2Ox!<4w~3{UZnM`Y-I7Nj|k#Lc6cfp5Cpa z9lgJ2Ci9*=N2uqs*uApNOUW10e7wWud;dY1SI3zt!qcIE-B2|!xMY4tc72d*EH9d# z0Uod8{#ec2=IRVSs=z^Y1|LVVvE{*Kbuc^^*;vztj;Fl+{C-~(ZhnR`ORFyHUq_#x zp^WZla{m!<_zb@-)+EPgazD6yM)H^Exa4iNnWFi0^bEc>;M2W@EbrFXP&e;~@X*be zE@f<*s8_IbTjA?Mz}!vWG%m5*OiE+r;0hQuPOX`~tL{UAP(v_1xJ3L}vikI_XnLei z_Fsp1l=!>mp*Jr)9qLNvs$VA?Py6qp$9QQU;{ChtqmOy@c=DwgQmd9glNzE)yf`RB}cpME2o=Pu#$vS^yI?g-Jwd@H}7cAyjN7IbM3Pkuc~ z_>r$Ha#6IbmpV*Wq_$`N4mNI{D zp0e=a7ReFP^xOj+qPJ@V<5j}1X?&LVS7e@{)kXGICM9@A^Vf^XlRw$z_cL{0fL?>` zCH=Ris;cKNK3df?7QIa)uZhb~GfW$Pxyj<-Cet$VeGT*>`ss9SzkPPojmctT_eX*C zlNs94dw)I9v26Dae>)z0E?>r%g)ZjNZwLG(Q3DRL&D9wiJpf*0>+Z8NJ_zZlaq>86 zbuKR|&N9zwJo)V$dUp@>h8-cf4-9j_-vN!ubn3Gv#}6>(2krn@Z;|FN^Ca}Axvp}Z z&}|t1bnC+=b&53X_XC=n{`|Dz=oSTB#U~4?(}C@i;+y#q)gyi^9K1EkRs~(IJ>{cU zTlz!SMXvtvX}5o^^xrHpDf)SJ3;on{KKOUqp`X&L)lN$W-Wc%GKm0ugby)Opw(dT9 zo;**TwZc7Z(C;Nq-Suw!@aDW&ja#QfgTDDUG>(o>(p_s1e1%OPOy<+B#&i|G7`)dG z+0F7JTBn+7Iw#r9tq;2OGj8RHmp__X0nZihPP}|xYU6u@dcy- zj<5$Hk5PUl7r&z8CnrTy`1c}j(Z`+R%%YjJ*OoK286J`jVV@@QqNx&WTRSV7T8n)C zI%5hv&i#GducDve^uV7>m-6Mq_wm!`w{JzZa39hAuV~k?(a^a@)5h8GyY~1|`mJ@s zllZ9%8C%Uk)ulDU$9;8oQFkr&GJeZE>Dod1awL;abM4dP@h8zYJ804?vrYMG=!SKMMSb8ikyFrYzxAbq zA^3hbI`$k38b)Uje(ysLw&u9^T375ux0g>Xyq5V%e>Febs6#elE^;)7I@)NTxmB}} zF`V$l=)tdrCNzJT-`6I_@g>aek88{_x{UN|=`qVl+jM$xj%2qDtTysYu=WIt;4v?? zrddZAUJw44A>G5BCqEv-$C)**e19jH<@Eg|uxK1KE{Cbh#Y-S^2{<`voAO-NGA1w% z$LPa(Tzi=_6PYt@p^439;G_)sc!ibQyhnPT2|4t(M|vLjqPO1Q#7F{~!?zS?t>TarM?_=gr!z)!{z7|d#Q8$6qC%6Zb+Ky&ZV4_ zM!rLVoQ8hVj)7y{e~tSs$dk*gug&;FFcc{RpUD3Pdyz>7(GT_R6f!6=Wy_3H`S{V# z$EQC28#8)mYn>tbtmy2y1?E}l9m9CtJ%rZ|{L#byCU<^*V?gU+f&B*i+2NyBR&ok! z8^`WtO}ENCJB9U(6IB-11zZDMtDZeFEvxO8(r6mLY4QZ?7n4gnZ?YG%t}!cqQ}692 zZQ~=iyQU<2;WP5%cRv!{_}Z*s+s&+f^;T6?9G;ckRtYV@Oz!NxR{7%_dT((296LOn z9hj1z2R6-9kAFHgArsZ151$-j4z$29Rm`_37+&MnkwF{URo&#jDiK02AwMf=cLyG@$(uw&#=8xD{0*}O}} z!2cog8u*~UpMt~FGk&S}zSnW;P=;h_>ey9(8Dm#7Qb$1tYa7AGa-K% z`O*jchPf!-;~)333&h*tfqTk!4b|t!O}9S!9+4bSolpAeRDEUC8>UX^py4lotvuVf z_4~$M@Jeo$U4>mf+3cT>-uz*c*jy4c9XZ%0Wm$qR(kvS!x4`AIx!9Kzzxc(2&yxOZ zef_S>yDo3uVoh{?v<{!mZ4jUB&hV%3Um!optNy7E$jiL&NL;ji_et%7i zZ8|pc%Lax$rf6;tHhu}u`TS}FCN0}cb=d(k_b~73#+#1C{I(UwQ<~%JE;E~23*qgh zrm++oz7`mIuo1V;G|7%XBD1Mumg>Nl)k?ok(eDzc-{nL7MqjQ`yD{nPlwHB^ht#h+ zwo?cC1@t}5(F^XSJ#3PKTE=!UKlHou3Dl?h{kV!T)`Gj(y6}N8^%rF)X6!;ALpHL0 z?fTV4r(^MP;&?Vy@{97D&+mi$ZsI52J~ZDof1yv0kBCQT4p$><NKwJ=X-Ja{|m#hE6ak! zl4Z4>p3Qg$O>*1Y)U!Ld{~h}F25FkRC+VMb?RTC2Nk7FuM;~{wPC}4C?Y>#A^28=`viRoU1Ex)w^g#ft1*$!RzE*CO8 zuVg&!$@nWuOvh#17jPfpdIRmvz)uoK2524LsehJRe;szASHGou)jxB%{uq2z^~Z`$ z$28TCKc+R?bQExJaesK4NgpmY>B&4FX6?WD0?G4&_6YrxU#U9F%u%{*I%wPRQ(XQ5 zv*exPXnH5Q3HIH_LUg$cfvY#e!|YHYzBc9p`fbsu^g`0}NfR&hc-_`OIBD=Xf8sPe zWCrV^;7>N;TJZVubhG3QcyBARvi^KibaGZsoBA)@Zk;;Z_e$N<_xc&beQzybT%f03 z*5F?cPA8@%UokDB*~npZqMqG>}AY8STO1M`SF|NZ(YE+g#v%|zj_*= zM{sodtEM6XjD6P@)`A!}{FH6jzFISso*aQ! ztE}EFv_j3ts-mf@tj?QelP?sw5*dxJ~l`#lsWYN-6< z;F7nlr5*6b`zbFnRz?TPvDZQGF~+GCUY?{~ z-3vBoWoPUX?n8maG3*X6X6i2;nl+2N$D27r?{m1PTrYB+*ejm85&{~_uNn$%CA&Arf@c(3xug6=%% zW{vykIxFegp)NnT2iVZh@&>5qD7YK^glQ9>Kwrxvh69hPwp&8Y)9=~-D`1l^<3ErQu4v$ajc@KTdqy7SXiIcg`C@|@dQ%AeZ zdXIR*X>9G2H$+p4g+7JMxE{Vxk4>&QUq5QjFC<;_U2{D$C7vqMT%R4ksW>>jVFGI+ zn&4tDInXg+jar}@(HL*`H?&xtW~M;m$- z{OiHlp1@Vt^~{QcHTMg~u9 zKXsRBI|@CHhc{+1$9m{{75P=S$D=jxU0J;6eP!_*A(ys_Sr3tqTE1#m@9<(9hy7gY zlk_UtMzQgx{3N>MV@}_%ZSEasI`CVTV?U+hnKVzYmYkCA+>H-Mv=}q4eq5bha-GVo zB){s;r!SH}rBFZ*?Df9F{9&&8R1Tbh5K zU(3^D2AFfF(Z}9Jmj8(Oo$=C`j0zw$?aFPFL~!+Jx*ZQR3SBR#Tx$tOO1 z7yRsYa9s?nf^QxA(OSxVwq@bNFC%B-=-7(C8RprA?Z2k~&N`*|F#Q(nl27PiMUwkR z1^-NU?E9D>x|e*iC7;Oa^>aJ()ssin8CnaCcXhI@==hFKHqkUz>7F`-n?CyK$YXq2 z31ruQQor}CHhP)UZ@|M`URLCj3Dev;3op<3lb`V0b@GF~0sqt7b$Ott=M0zoAac^9 zt@qQH-?QC0>iLJtAm~f@%|s4JCV2ATFTv}OT=4miVh$f4msD(rcz7Rk7(A!zuz5=0 zlb?Sye&b=r&B<3eKB;$O`H%EI)HFVc%_Lvi4x7C8MW)TQ$Ite|{5kFJrrojHQa`-% z_Lt27@MY}$VgHr)UVW;iFM=DNX+-g^2+X+_vHp&|7xuN`^IN!lZlquOGod3(BG-$!%C@lZ8@9{}Bm>YM_DlN@&{<P@Xc;(u!hXzMpJ`=X zg!WS{=x!~u3^<6UfAGf5Z99+N+V;cw=&1Db4dya+I`eov>(mM6r*sX*Z%J>)=4#SC z^RJC{*P(T4=D2(iD}X5u&-P%P&-2guO$TrP&h>oomQzew{2cUiLwHYQpanU4X3fj3 zr-AESK6lACPk(a9eVC3W;2)6hYi+=!W@iI4K0Jp9!XFdh1*Ng59GuEttOWK%DLlu@ zan{$nfx8TPNHB)t!IeC(q#myh^;>l~>+_?^!oT!RYvDVeq;Ca`llXzk5Bo@n8T8TY z+5H%cet+c9uk;8{UjN__d)n~xrJ$u#jML7=S#95BtWJ~XN6@zGGjMfi@nhyxME+^5 zgWLGywC=qYyZZjTPdi%xv;V4U&^{0QQo`SlO&dLEit z<4d!xBF7euL5r)#NN>$Isn3xvdQ%;qO`1R_MQ%hKT?zf?W%6ue-fd&vEgPdA(fX%H zr59oA-btG2oafXF8$Ga$`LvC>A>Fi$dDH{nUP>PMY*ip|H zV~@aR%QCp}VgxQ8#+4Hz;Nj}u8Dl@pe@DG%*8J=H((0q*&88-PXZgE)drd^I;uYd= zS|{2XOZ94Yoi;_$KF21`(ND}-6j9lQ`pSE;W-9I4~J~iGlhXy6F=PJ+x?>|HH~p%@DJ zF#nzW;`{VFy@hn;Ij6qs()O;05YgOL_RHQ{KM@A-W*n6QRp1)5L%8vnzx2 z-Hoe1rkMLUHfKojK(@06P>oMT_rSfRg&5P?OswiKeR=-he|C7C`YT^Q>l1fwrQcyA zo4#ajo8&~((t(mvvp*kRYnKkiGL6vJ#0Lg*O?m}o1*g3pe!#rY{Bq((co%~%s!h># zQTTOvz}$Is8EYo%0&S0i&r`p&+9b1DEMiD~GE@7uP=A;5PQYR@e8<3r{uk#9vF<1KUJgm6fTZk8;)Q zL9&fW{SEg~?*HKEjq#e_`}9Z8uRazWzw*KH#dE-M4B8A&?VOj8EYHAjg8CPloW0@t zWAmzkui6Kn`uCC#wtF+MLD%TC%uB9g%@6FWh-BQA*m|Nzjb8&}zT65X*IGIKeZk!J zPH-5`uG~fZY=dkh{CWN0WQpn>i_XK9W`B5au!D8&^s)B_JC;s0Cz_~(Sg3RfWljRi z(rk00jx_4X6aF5=H?WHJSFKMKmzc&nY@PZplf0}l;O-%zY#BD`d~1TUf73?CScrYI zk~MtkN8Q*YQGR=D*ItRk*Vbq5JArv&=H750%G~3_ifGLzJUL!d%8wYdf^M!|{AOxC zbFJeCZVRfpR`R=m`vlh}eu}@^gTABoo%S;KPJ5Yqr@hR*+GB1lX`K?U*~%|O`EIUV z{3It@sdFno@zQG2D*46oh?isj{vnv7eM}?f+)CalhD&(W`$fbJ`)#NP!DlPDO%m$% zb$)-t?;HG{;J1O_Mt&Uu)47wM;jfkYO{KR!=slavdST}I;^95q(B^UC8Pd5naUWAn zUcD8%-M(zkW^}ukzeioMOkS7ne0mmV19m;A#UUKN4iEAri?wpl3(24AF^A_Zq0 zX1`?bQ_ddV&WbSki1q;CAc{ytS_;9Us33EH1ceL4I@ z1BnYe=iR2YT=ojLC}xH}H$gwQS?0P|;0uXyofS&cJOY-aWRPU~Ti_#?I&S--wKxJy z)Ujv@V{p-*tC!H?hxbb$A@^!$oS*o7$DKBLsvd|YMVc9P#h=w!j{_nucG|80L6&m|L0Gjiv^WB3Tf|KLXlc5pAbr7_mC;PA`h zk$lvRJ(5nh3O`UQ>k7=r#y-a6GstsK-UPY-1@dMwb%xm!`aJWaZh@72w8Y(~cpht( zF`nB<4_j7R*Yb2Uy%M}=e5x6nmArp#okTY*wAB&t0D=XfL4bi|l51x4Aax<-CF3Rcg|ENKb@zgvf;M}I4f{+6k9vO>_aL+280jHksJ5c% zes~rt<$+UYr<-K;Qow}TL0CdXTXX()H_zc^p9n_Qf zM@MJmHeQ23EciEKZEl5@g(Jpdr|@zO^>WSA`n>v+ht9`b-pTrYo@^Px0|wLae>7>W z`K!#ulu_PU&b0t|M%tT0bh- zPUbAvJ^TZiI>)kz_vQINk?(%kgD&g`BEb1J@U&C!iSQ#NR{*S{X;-xmJN zEjM~xH6C_K>$q-AS1tWnEd6s7T>m@@u3bEzdz^;p#;JejpRWHqqv~(@ll8lH+*RCti(I{d?f7a zpC%ae8>Zi3ybQpr)J6#Y8|EjGzk zQ@559w>q``oAl{<@df0L){7-e_q_aS1lS3TTS^ zM)hTF4zx=jR{L;wA@|@kF9)33t1oo+W!P5nHFj$-C0oko-Q(b;x@&w&@@ljl;EYYO zGNrQ#JpZFNo|-#-wD)OWdx1=Q;+;cdTRATI_U}f{pXA6nxKSkN;G3jTe#4vl!n1da=9KUfBLJXMlhQRFn9Do6sH6gLXd2 zS!8;ze9n5JFAqM?O(%&T{M@91X2B=@`Tj}hxWV(<1xE|xIN>ofP|Q91c$O$nFEVRA zeHf4LkK&2uxoee1%+COR z!ne_3-T`;;h5gc*cH@tW0ei7k)L#qUY|1?Z->@l@;o?NxJCTWa$% z%51R;`}G{*K5Bj9UE+k6yd9jtT$n(+Ip|b5?T6ttH%xcPboOkJkl9*;zg2it?Nv(b$6hx1wQM@ zedz)4GDq$&L+)3ZWNAk3%dhS6w%QEdbEq>!>=?Rn+!%u9QLrSukG^++d*v0*yuJu7ue{%W@c813#KDX%qjvoJRRm8yeHIV8D-+*v7xPum z|C+h}AJ)RC@Cg)D{lbW>g#JPZ3$)6&h&(%+F9Hh_nqIc4-twJeXso)^*x7PZcl($QO{U&+oDc<=Z-!*SY`GP4sJ>G4jxKypZrBo zJdA*oN%*pTc4wK1RxlSTa?q=C7>8VA?G1mxg(p|-Ez(&?G5JK9{OVO z;*u}kcKgWr;f{Y9bi=rHbmD`%p8JI6)g98|?YTYBMlJq@TCR`b^YHXxd>>WlTl#Hc0c)$Q_jhAEg@GU5mrg7&73i4{}VX&{|Pt_GDdOmI2x9(!)v^8 zapCgTFaO`chQs%LakQBDi6?<=BYN-MT$}EOzuzW4#k%k$WXY|_`aF|npAKh}@%~YG zGVd6lO|9)!*m?GK(k~Y>POa_1)cS3*Z_nQIi+$kqbr}CH4QqWi#&~P&yO|H8$3Wwc zzMGCQ{tKa%Fk|hFubrWX81xWlzxrre)BH<_hrB*)7peU-Yh2q}@9eKvI(}K=Vk-uZ z@(WrOgR`s(8(4V1&n~?nsd#2|@u|+!U|Ef%fxxN;h#(s!G;xOdHboVQb z#gEkgvGz_mI4B*Du2|I0nu}!JKIsj)$Yif>jPJZM^boai8M>I)&hH%FH$k@P2)*WY zbdpep_X*Eq`8fZcJT2f$e8|H~H8{`d%F`KgHM8047pq^sOL3z;=)3sKr_A3;ET;6T z53}ZQi|cRGT0dufUoIWGbKo&wyPPcLz=t(|&htN3`}1euG-I3>;+|*a@sg4Lxo21#PcWZF2PL=Q z^XwDNMN>Z~eKeo);Puxx+Ftt>7VMo7JSJME%`DJy!8M(ecWfGOF4c9`lNRUHBPj-dW<&R@pt)9?=ol+KSeKk ztnPbfA@882-dWjY4({-0wCNeWH^Q~=(yX$+-R!Xso7B;DtY2Izeb^iib&Y5LIeXNP zqAzcx&MNIaX0H=z9()mG_%WWVZ*kL?W9vm$TdP$r=ds{w<(Tjht`o=z<-#cq2I201iga9mAj7f7i}X-+J&d_R?P8cLQbXGktZRgJmK3 zlyE~I!SQ$}Mkwy!IMn_y^*d+b5(oPQ?t6)WHJ7u-=-po(O*eY?bE4_{z59!!=?3q< zO!i==>`l?M&Lwr~sU)7(yFXl+Udp}W+u1#Ak31ketpHC)Nn7XQX^w-ZVOz-S@96X8 z;>-=nKh2f5@lQBo#5_^DZ?5dH$CxW|@^56WtN_=-b2ZQB;xohN-n#q-JnryxF6lYq zeNU3sayPzU;_`dIt-(JEjyrgM?o=?oS?i*Dj}ErsjOH~o&)Ud=BGRqZ?s+|ZjJf2v z^8EsSD}C)v@*1v_!5wE7PLo{uJ!zek&(wFIk7wzV@9s>N~aplp8;Y=Jkvk%pl;fv-wv&!UpyDZFQYgZed&>+hMq{d;D9j2<(41pJ;p_~(z? z2HHEs@SXPrq z_52m@e&_TzzrBhye}Ctz9^Y7MKeWnxk}@`PApQ~i>EBS+>yM}N_3C_O&Rd#AdFJ>I zQ)ircR;PIeuT#9!m!XR|bd`C2-}B`Beua(HS4%&HAIdM8L>uVEd9l0y#CcHoSv~k# zm`}aI_y#-Fxw+V~`q$y(vLCUYE_26nUrXLC_-@RJYpuj)mv^{p5sCq3?cQ1IKwp*p zw47W9%$Yl%Q4niI`rX_Gg0*7 zBhUf(+-IiTlEyaJ5#H*;#r$<(Dskh*6Vcs-rerFS0c_>9IngwZfv47D7h%IB*J4wF%k)}os>Rfw zP<#<sT?`si%!v z+P;zcRW5-ZM4PLRLYq39B%`b6>HNr2>e2ZNw-QgIee(ZFygqxCHxjq8)4^%RHkX_z z181M%XIO8!n7ZqMS?efmtV{U&`(y_0&M&RqxQh1QLjMumEu-Kr3yux~*tMRZG5E^=aO{{@f>TP zrK?j)Mrd#IJJkOi{r1)tKhC}UWsd;oAoM65CI+1it_`ppLA*CQb^@QslGndlqd2fr zv&?G60S(@6D(=N+sJ!sDozlBm`^?0u+3gP@0~9ZJ2fXf-_Ifk7T%ADsF^D4~|1Zg- zIrcVuDMm~YvVP~OPb_F_4HWDRu^u#+7^dmed5p5st7?hOcnKM^Hc;rq2VX>*@~dv~ zCcS^bc|X;8{}w)i^GUz`vFJdI*eb<2$KV5slR^f~30t{)L!6acp;#$vYJV4d99S1- zjY(&HuwRos>%tp$i{#LO?bPjWYm2WfaFr){zL_+|pN467IdOMdFF6#L+J6jtW`MPe zPI$wbK;hn(z+1HuOzaaL0aIY_4B%O4741FB9{f_^lMm$QoQqrwUwo)x?!c=vP13D1 zn!Ynw)V!5*!2}=vzx2bSV0_RAqu_ghXY%6D|CjgO^E{USVn2KDM*8Q*UT@N#n{N1c zA2>`ybMonEEpuc)3!NCX-r$_}LDr%brx}O$O4glgZKju)v?6dLe3a7PDNbM6zlFSZ z&&QaKKS*c6<+ncyzIuq4v4T^<mPIJuh4KD_A`g@U+T6^63?%IRf zh?DGb^_K%vsY5U+9+Lx^obwdy2~(TnGx=t~3{w#F#CqzodKf9%HW4+L&nQDR5Y93i`J)Uqa-c-Tb={UC*+B?DzohbGX{3fTJu-8}?V3ES2MO;yYynkjE?vJ3m^pXc%a1MAVKPw;bOAj9Fonj}^^@yk7+=qjKq}IpY zz7E|3ou?=No0~O;#VL31yPbtT1rOG<^lRH@PZaG9&zFMsiQs{C{KzLbGp9GWpu)ig z^W|P}QBC}aZH;r|jKHhtSa^-UGdQPxX0$D2iJdG>otb|ge&_{6(ipVkxde^0^x-N<{% z_TMq*>fpz<@C&;dzX|g-bTeoEFrUAT?NJNws6%gfiTir)!>k>7YXqYCF#f0Iv@4v5 z_l>o_>H0~A<+$uW(PPgtll-}8m>Ao$=nvesimt%zSoHUg=rO|o5dDqL=cm6>?;QH8 zg8qP6{9q6rqL#MAqcn%5<4eBy*GqqOW~_MApR~R0v%5T-fwNa4r{JB^p{uD^w!+#k zS#7cr=4njOHQ~pOjR0+r$oDX|R&{3YR+Ot#7X5!V zh+LBH_=I0~oGp8fpFww=s(R7;wGXeGeUP7}jn|239b3;UjC_`QeR|##VeAR*-x{sw zy-lo}|2sjhe{lF)WZRDF9I?=T2ez$lte@QRJYS^%c ztVND-&SvowlNt@WX>rRWCzh`G7n2%0kr&nCf2Grt#lM<_Y)NcZKa^$;7xCS~%>C9P zXaAm)URUJAv<>?whT@nb^1qAT&>NmwPaX67tQ0YOS9j4)7(9o0UPvFq#NJ5u;TyPG zvUJ3|N>>(TWIOYWe0k<-#&0N}eE0dt;@<5UT7{1NJW%r8i8=PgS`5SMt;2F&#@!|< znh*`VFdZ5Kwh3C(SYQSohlbYXS?LDytbNH0l#y?3p-ELwjm~4;Zc&WAn{C(_k3$;` zf5rUd{pfe+;`h!s2gMhICfRyK^ZbJSl25hl(+;CQF#ZjloFxFh;Y%6W3~rtI&e@wb z@M?WF3=HrUzAcf}xE)%{g&+3b6|LZ-PZfQ4F$d5Ken-AYb`%o`-wye)?~~B0=INMx4!RiJ9om0p zjTwrMJ%VjnPOQ&SM@J4KlLPH{qL+%!FBBhVKFAig;RPY~-oD5;Kc5&$F$3yn3c;P+tlUOo%Au%$H9r%tc$#de{o+z~Pld>Ii$MKyE&ZckuQts2C`UiLM zJ+p=qcqMav3VYNWXTi^Gc<2_+V<}<(dA0ge>hjR|2hC#f&|3EIiq9&KHL>i~@M4?mLW})XmfaAeeVzX^0r=VDH>G~T>}GhzBJmEF?_KEdwhtp~s^Qa| zcRojUh5B2+Akw__Ot8_-_W>tg75%S;PiKKY)-!5q>6iFYhwy(atV{t0$-J@DfPFOSh5XsY~vo)6KV zZu;{dz&4nj)yP`%lI07oY+hggSMz>OS~Yz259*g?N5nH#R%P1oGsNgyD?Hlcmp_AF ziq2MUGiAc}K5Rh7|A6}Q*VmZjUfxaOY^$FU6W$t`^fdTdB!16_=^t8uv3ZC+AkLn% zi@<|?ht538*r3o@@<0z~xD1_9IqW+hIqyB)#)>$;5z67~is0*t==?p|Wy>!#i*+V< z=pKz{!E({lE8;ouIUAl6`p@ML&9|m5_xQ2KLi6Tax{LId(Y(m;NBgMr@dN89i{GW@ zRmM^2{`nSG+{sp#)pNZu_m&)#|^1r%S*Wc!UWph1r@gey4Aojo;_{beRS>rSDeZ{YytM6K$_4l21sU^iG zr=iE!@3{K?P*I-NPCU98{x-L(^E&O7dhNMqnXbs_1M}%y47P}F zv0&a8wC|vd);cuzRmU?tkJYB~I%{0UZeBTjDFrNyVZ09>dPjX+F%~}fQs+vWG zPis!}JXSof{omyGN8m`z zjaEGM-uyP{CEr4}e)}3zVNdx$USh7P=zb~EypVC{44_DH+EmO(H(3V%+ceWkCMaLb zdQ@FzT_+pb^mMv&t&NODkJ9J!6*udHU3>YVKWt@lypXZ(W{gGShA~##Tj*aMaHzdE zz{6Oye->Ri@w~3@#nJn7qK)fl<0;n9!r;b+pWVoPiL<^l_V*05#?iy?HpWvpe-S(9 zThJ&9>l1xE2;zd!B;^kKE&n}jomrX(Y0ZrxZmHM+rGy?h`S8ekvu)A$RI*ZbzupOxQ7 z+vgaM$O^ub>9ku{?VJgC1UaKK0SnIDdQdv{oTe#B8(4CHqwtG4Pq#k2a^Pj?ZsFl* z#rmL`iw_}-lzNRcORESgY%J-xn|Z?qf+j@(R{ zss4JXwnW>nvi<(zrsj?-v<;C|=V#g!XhFWw?t^?o?aJfkG!07_d@2&HN~N5<2PU8JjdtObM`@gysn*L})jXB_h`o2~Mte}6~e zoL%76efNQD4#5j_|1jlj%gw94hS)Dv50442esEyMbdyp!@Hu76B-Yh9H(WIKe&-xI zbmd!L>H$ysegtFf(BTg5SdHKMKncCky`Gjn}66$p7^4Z|S<2_uTWW`3lrxRmb3z{xSCRo#`RXkJ1 z*Zf(02eo;`P4n~8mMV?0P4M&5KBY9ymDIVEUfLZ>Pc;n~QQ>ePfQ8zy?ZHm%ZCra@1(k3YF3exy_Y2%f44Qc$mv;b-6 zhHGTJsN>ILPd586pofC~w~*;A%&!*s%yD?K_EkfBOP=Jt^x!z@$4HkBp!Ah5S;ZPoqcmL&oNBXKj_RzameD7+v5&y???LS1*|DkJpH2pKKD<92HuDW~N zP?>IDnU$okS}=a-U6=nI>8oC{hu*#5e@FVN1vx|Sp7*~aebuAG@1F6!tF5U_xAI*3 z@yhf+==!P3^mlaqOl5i-SK)uO4erj*k@zo;o9AN4Y2?%id=&@st@!4!Tote1N8%`o`GWwRGy|LiD z+qIdTb@qjvIk^yjV-Dw}u=lb&%ylXLmaWi;vGM2eD7lCgvMRX?MsbF^GWN=*M1L(Ia;f2+#2Ntky!})QI zW%NV-ozf`@!lB$4P2za4=lBK$y4`PB`;xA?igJ=c70_=h@s*)Uogc}!5Sq4Z{?U(~ zuBk;v7n{i^u+f*;HzGGI)Bj=m*L&!Z@1J+(=7YZzoGY%}%UnsKLv3*Zfg|F0kR}j&6D&h`a*{8A9)w%GO zM)YkcckR5$@e|6ADLqZRTzxr6d$GXMUB%Y<4Rw--3(Wp3=&pL1yPkDhaP)dsL1ZcL zgYV=;^yl>K_y5kyF#q?~OkjfI9{tQfg8Ml7iqbwt+NMWcJw@Mmt=|w$ z_mS?-lj+V_)>21FV|MZu@;Wp<$`&{Lx%2OAJ6C<;+a~FHmq+ijk!_+6$G1ZtTV)@i z!{MWxqkeYMw#}L`FwXfd`eZF-=O9mN4oGjmMuS8k8$P$eG5B$Y`U{9(Zl$} zKI5K|lAxVWyE;t!Co=V(t;d{Ach@pcMSHFP+r?)Txc^@F8QR;X`wZ=E(LMN5p1fQP#yS@ z;4SW2k@nlH3o4U`=Qz8?6u`w*_TV$XDj=M^_M05 zcD@)W>Nn#vYc!lAR!07^!|dt1)M~@mwePMbE7`$W<_ABuazOQM1ZPgHALZF^Uoh>H zEBRbiQ;qy#vUPO^qU5DR!{dk>%GeIYe`z&#VM|maldZa$$s?J(n)gO)GD_b%oPH(g zQTV#x2?LK!{f_@!#L=t_tMHX^{MV%)-x!Zi~m1|{f5zxRy5ax zgWFi=vB_(L2aCPO>0dkcEOLDJ``Q|6L-OSJg72M-w>QF^ilh@ucpnbuc%Cy1US14SmdJs{vwCweX$wECBauN&YWK? zy7T-DPMe2kx@}ssl4=iHt@$MF^?-*z(B@cel#oYl3}$=;hQF|IcG%lXkFQJSW8$ZM8~-TScz+m2tUHF4UY89fwAx- zoy38w%he?`ehyC}UcetKrvA9_`1~5a^O&`Us=4UEc6bzQ)T@C~2oe8)`R{&Dx_(;xOXIs5bJOX#d|knc`9R33cK z>63*o8@lw@;gwfEwI-*&p6je&x4*;otc!P-4u|o6k%RYI`n8Wa?8mvrU@V-ACbzPe z#Gy%_4IP6XG+)H$!c7+?L-_taHqrjb`_jXZB*^!`)loEGT`)Rhm< z6knOc{9U;%kgUZf>x7maz6d{W&G>-UK3Y5=o6hpt_@ix-VLBKoZzZ3+Z=SF}BeU1- z752In_P<0MWyl!69Z>)dI)Fhk;cMVRb6a?kzPj^Eg-;zgOcN59Bx3D<+@34$Y3z;WU2UymsiJ)TIZFz3^hDWMc!H6FrVke|YZdT`}*3 zXU{GfjbE?7vTJMcw`vbZ47sDZHrAMHJQDb$r0>h$uQ+#IgokQ1{lVwk9zjU(7QNWzrT*Ulu!5*%jj_r@x7 zd=Jh&c|H2{)~}8(9(gWejDG9(-;io!E`%j3KUv9lCGBQMX6btmR(^x(FLUhk9A}M*n3Ds0@JCF<&*8;~eV;Xr z-=bq&S85#J@{7P50-tL?Xd2^`kA0kTP*|&vIXY^7WCOU;8H9Yd{J=`)Y%P6E;ZJxQ zIKbJC=@#GkCOwSJoCB@x3D_^sg%%EtGbd`{1wX*&DSY(GKQObY#=iLAj0FDnUHI!b z<0t(h@%+Mt^urUtc|K=z{D|*E{WtdszTIF?CKgEVDX)HqK8^2)wsWS0n(=M#i`7-8 z7lH?!$Geo6-%k3lG~1+a0}rR^r{L;k?|=vU_kmroNp}a|HQynRGZxgppEfPpH|Pz2 zzz>3Mv%{}HY0ZW?gB&~io%)zyJ<;o9_{@0vnCTm_0WUaxmT84zPU*3llnM6d&UEm zTtk67=f?1dQl_F08&c(M%7?tN+1BE2bZ(^^(pekS*|k>-pK7lfTRX&hD*i0zTu;U1 zLtl9zp5v6c3b;nk`QiR}^6`btcXBdkZxSz9f-gYdt%@<0`RkQV;$-+{`z- zh}$ddZ_yeaZL~7R?OcJgV+d~gTrD5as6HPC?mgzt>-=!bZ{H98+rdK{eQKqy?=3UQ z@69sl)=N2iYBu#>$vv?{eE%!mO1$TGd_LQV#oW5xIA;%ji}|4a4fHux4BThd{Pe7| z^Syq1=L+_rTd}@X(F;H7{ceCgJ@J~s!9ZCrZTEdQSf=wlbl!;KeVsEsINSL9w9{L~ zSaG(o*3^h$OO@cO^Pf5BwKF=-M6!i7p_IP=A-m1PO%wb^^bqA5tBJNvwF+)u$=rAm ze<1uR^>yapLyRMFB<$zYcex7MmvFW@?;AS7r-fcuTVs+h{?f6V^29e{w4*lkedYvw zpc0x;o-et1T=_dn7a4-vk%#V?G#Y>Z*;wks=<~72v*TRnV?CeoyKVz-l#OUTO8aRg zSySSBf<++}If0vko9(-ivtI>Mo9h@nv`9AtwX~ADYp07d& zvYm$dVk1HBK|4=gO5@$_uggkxG?44D(v(7|fr zG+Fmd%Gb5xZY#;U-J%YBU52`)Pn^q!7{-&w4_v?Kvy9K1&{-Im6(1wLS8zTE?A=$~ zlCE7I8z|x147#u9z8xLCq{}3u=&qX6YX29ETPN=~fQv&{WE03per_=kL*2H6o)y~HDjI7gYZ<>I}I@H5pizeej= zLwNA%gOtaIt#hQU_MFvs54d&boU`uh&r$a|;XQ#Z^(Wvxfj{wo1aH&$KO5d(pQCP9 z-i@RO&z{g)hV;)nLZ)LbzgG4_d|~Hv58IWl60e41Z&?!1-Q>WSDg9lIP+xuBE=}zk~NPk5=L@ZDp-LK|5j6+qg%T z5Kr3o(E8xO+$v(G))E`?7u0_!5Nych3QQ+nM^{&DoNQ~Ib@4UU6?!CZB`+y+=%up$ z-OxtsIHxb=N;jkz(1zAy7G}dEUgAs*&T?ZuKCuD-n08WtxK^E z6<~jQupQ%N=V4@agRia0+>?Uuzg6&?Z<`a#&`ZOttFiZmGp6liQT?>1 z-yYB2Zg|FNc(xs!K|JIyH!DtZX({{)UAs6qqdz3OG%$nsvZDS|_)}ULFUdFY{y*Sb z8LjJyCRV<*Y4b_!YL!upbU(hAW9)z5BEKo+zZNLkTWxh}pUM99d`~V>ADM?=ZyvGF zJBxzT_Nv_&x?ev!sPe!s_xJPN^fj~0{@DYAKm0ZG@gd$FM2=LSsXUnf#cA03Gul@I zr(io5T>qE7Hvx>QI@`z3BrFpM8U!IM%7k?YM8axNlt}`Z$YM-_L`6wR1}0=B3kEk5 zT!>;VxW=UtacQ91V%0W$pVd%p8`QP|t)+gp2EjIp?FTA0RFvQI-1DB9JCg)Z?DGBp zUO0L0bC&nK=RNy!&wK}Qe*{?$z|WF!%$I>z;2fwAzjc=ic`*0bg)!0e{6oVMaBls1 z{muU1{SNFzTC~plAK>}*6Xns<$QOPi zWbz5=KB+LcyJG8GVFO=#7jyVjf0zMX>gW!{^DgAu2fsRP1Mof00896UxNoAXavR;c-sG9>kbF4kFNbkg!g;xz?B}{@iF){ zegnB4@hu5I(^_|W`&QMCa=5V)bq8f|Bg-Po_V5G8GXH^kFcC5m{~Ybp8{)Uh=GWr$ z!7Hk2uB)Ic^IofN}_C(^n z3FAw9d-EaM5dKd1M`%Ol<2NPI-af-|eS`dVW82YVQ@DSA8_GD_N!lO4J?1I=E>GO@ z5jX3@c=+*w;w<x}&TqJ-OHng^#iB4YKu%y9{0aSj;g0;bdN0r;iAvG7lS?)-v; zB^YPaK<)_K<4nJ3yLWSs`Z~yC*W>Ts3d}Dt=CjUZ2aj914rbOf^zX*95B?N+_9fD6 z^5`*Zo?5;8Ea*A|dt9uqYmp|7mAhdb<0I^S)S@p#KQuKPciS`n3qaZSZ3^pspaHV_ zF#mKqBCYQ=Qn3u-3*#Lzp1W3rMZAV}kzL38E+ce*?0EgqB<2a$;r{{IJ`L{{I~(N~ z{pd>0D?Ue=I)c4}!zfpuh4;W+xIJV43fXFKwvOlD6S2;U`!@>ucT;`)*WWYv8us64 zs|R9Cky4KPjL@%-MIM@S6P%wV_PuTJKCE%GZ^k}Y?439((vEou#`4y<_-^RJxhDAh zM0J#U?lzo<;Q0xR?>jv|@i5cD^Aku1&rjszoCMduX2=<8@O`cU=P-Qc{KRSMIlac; z4|n)#d14?dWy*(4s*jq&Zv@zX+V6MThrI=!lfm6QvHb1nW~}4!{KvicJ-+E-Y18fl zeJN}}L1b^0ISYN3`ObiR-p6+bq2U`P{O=-g&nn~%Jv-v+sn3mcmF}x0-Jc@eQ;=?+ z8{t~lmpGrrvU!iB8{@O>_ax!|bF72d)1}Xh^oI_D({&f*!ufJb2TVWi=|H;sZ{->~ zb@UnX%jD~&&=+-Z7HhwTjg_~3_Yx(tiCnUyxqd-(nd(nfoIMrA-j4#HsXe%F-+pj>mmrsjsc6wyaydycRD?$HV)>x#Xk8i9PNSqyKu_P_01EVlQsRxj}GZ?4_Loi zao3!^O+(wcbt&2>w0Y^A>u<$3%i~6Rs#3b^Z;DfER9_$7=hfUcI3Mr)e%OciYMigC zMt&Az-jDWqSJIcqo;T~po!y_!wb^^I50`>{*MW}HIX6wO7|QcE{dz~-kkWf|FYbGv zxpmqNj(&S?y#8)S!8PrFfDW2aHnLUwv^Oy~Wj?=wv#r=KD436N1Z8GgI=ef&4fENE z0$<|?UmiQxjeca<#_&UH5ys8=Va=$GV{f?rZWr#J*5?NFdZE!5`r7C3!u`Oey*?N3 zi0iHh#E)}PydP>C?=BpRw73)X{twH&+2=eKGR_ZOS2BAfsC&`t1B}>NWIR9)(BO1NL*BuB&c;6Ls2$HXduA z+j&-=-z-pHPOP2qtUSM4;P(q-aaKNdFMdxUOidf|sr60M2J1T^tQ#;r*?OHRo#Qqu z{WwFxH3hnHUFvMe+Yhq3LER!Z%G4RvSWkQjGU)ZD;Jp(gzp1yaSRcoJ%0#v$18jWQ zFRSqOb1k{?{R8K}Ytbe#Jti%o-(r~bT#Pk&u1~gxhmU1{-iGzheYw~_xe05MSfAX6 zGR=PHA8i4-H{l)S>kQUuQ+BB5z4QAF z{uaz-kRbwkPfOVmPpxwOXkcIS zk|8+5G!EahKZU=AHi|FTnId(to+xqZ3@#@`6Fn3&l{T8d;#oR7}>jvSc9Y~#^ zw*UNWd~*reGEetxUxBhbr)I>)o{;aNAw9SFuof{2bqC?MCxD{7Pv^P=)_w|d@ICan z>z23ofxT}>+QN+Z;?0_d>wPPm(aF7jo(bG?DT!*u>*eiI(bEe9C3+)~IK3|w; z`diIy3C%pyzr%f?APVzv^1Gz_7V;x6i%&B6cmsosMKkWbheA;?9>k#x(2)*2X z*fV{FBWhYL^q2&>^P&I5R zE5dumdA6Hz&=b)8tlY4T9B(ASAKO)a59L80_8H3gsz~fnusy8lws8^8baVq>whPJS zyz_iOY$C$){1n>>d`~|m1^11u2y@O&8RprZ={U1@=IN;O$iHyjOA?Oz$HrpJiMri@ zHUxEmYg86*@+5Eah3x5xsKbVDc2DpHzcx^QYeAk^$L-7VzXm!Pj`AvF9Vy8BRK&ymO|6eW+w1AAwztBb>7zbyo+1|Q z(8l;(Cmuru+TX6%y#|2K1UG*78kFO8wz(V|`=k#EZFjlOi?(z3s-)rTF-}ZpJIwWj z^U$V+u{}V34nkQu+|#qY9eMaU%ny*)ht{s${bBAz{I1#%YyE+1B&-W8Yco+MI3L7X zJ?FlX<>XxD~0 z1{Ux;L&|01r3@pmrp3M>RL89t54K{Aj5Uf_>H>OLzo+G!(eI1kL!McusNd-}{YHy^ z{{bCrMY?uDx1U3|2Y}nKCdG7X{d!iH_X_B@mqV>~Vn5`bUeGV@I>0${OTSKhqdgGc zVLRa#fO^gq9a9FzZS?Ep83^0>p)>tfe?$6R20oWSznEG0 z%P-0<+LVspZ?f#N+_KEJgX*9xZ=?&qX&1}1eHcE(uUo7G zq3i~9%X!Wa(XG~NuLCcm+)}UM(5t_#-CC2rf5ecD?}48+enr3gXUtJJS9>1kNm-wM zM?J%jBXP=l=Evr+{&P*aZ^e2q$B-V$i~eYzur^n46XMDb8#wm^; zw`V(b#&Ul4SICD??E8fN^CW@qd+|~9cHSk-`55;x}pN=o_iGaQF!Wg}!&Q z-P(__`g_n#1I9;#FebwIZNvE}D;&G=TfPWipreJQ- ztlZO>-X!Rp`sTXCJJ_El#rniOxL0QidGT)L*u72Dw>sOW`8Egk!6G1&wcmujtSy+E z`TZi!i_!NJ?|lsOOoX?}_73Mp7wWBq*d*Q3m03~>*>70MVkRo|gH^>o$u@93)zzZ*7|dnT-RA9YsmP%n<4|8K|mgl8%y z`^&`ulnd_V42;E{9QQ}Vo@wlb^rb>SJHy(i9Y$SedK6?peOj4=Hj3je6At@mxZj5J zv}7Fznm*VSER@VSU0H&$0c}ZACY0&bRLqJ+@;U9M?5r4$`45 zIbUu60DRmSr*iFhEY?p`z%LcN_M&}hL;v55d#TZ{Zf`;whV|EaWc}qgTr8Vg@y>F| za=QX$b}@9vF>N`@0nTa<^yfdy*6k{OS}n#4!}4(s4Z0v7mh;HopIx6ZhqMo^AGXb(_Za_$A~W&sD{uoh}EiM0k~JG=jEWJT!>!N6Y={2V+=x9FoVDRPMd)`sW*x`O?S~=%*v6yGZ0DRt z;k$kx`(!h>NB+s&-t^N!-}SS9X1nWg3_gS7wca*<+i;IB^VPnlX2!Yt_dd)z3C_)< zu4Pf*QR)!#Ad_Rn502sPAKb;f1$1mp?cfZoSz>K^wRHw8^n7YYXWy}C{_N*I!MY39 z%B^n~xnF!ZyxZ0)oE!WU^YQzz4}UG%z|Rqn59Jo=pNe_SZs?@HuZPBHCwE)>-@Q<- zF`rKA`QU^5u_ovL4iNIYb$NM)Wh@)ts#$lK;QPNVNDt2grlP&-mFT(DsUAwl9Cnv; z?8UJkqD*0Zd39S4oH1WGVDjzfDE#y;@5fcEaYs4Fe^?Lfx1Ro|eU7{J@y$go_Jlvd znGc2W3fmK|>0;kuHTS(}pNjo`*10XHuMMcX$b-AAzIz0}DNNoHSQ8A~sf_xwdlk&d zrvK+X?h8sk;@I7$&+|Q_Fm`W6UNJ8dQFky$u+l)<1{t}I_YUX>$PYL6k{Pxc>lXZ7 zl4|%@-LS{*`yinf>jQyx)8P8fbsahXZuPB~qF!-*U@yWm?WWz5GY)rg%@Fx{(79Ky z-B`aph&o)hs8i zD6`DlV;$C2?dy>EJ_!9P$0Czu{85W?!STm!C?oG9Kg`&}KJK^~cg`Yju=WD0a%VZlW(!e$9|GBp$wm4ofG7FO^nf*w~V_5XIP9J-$#E-yG7XBqWcAwKHH}; zEc@Z?0r>VGWp~h@(T?Ii3Hsd%Kf2BC_|3SG{B!N`I4L$`AF1a<1!tlbSwuymh|kF4iFWs)ik+D4I9gg*&nfNVvjuo37W9JqkO;0Pen&avGK58}R0&L4` zvEMW7i(xC#H)U>0oV9W|s5@L`jaBvO(9Mw#urAaPp?0CY+vr~HYutqO+i>&=L3#?m z6TeT|d+U1SJ^Pux=$~2zb5Cp2>%sf&%=2nyDiRVpZa_cTlKnA$n-cfW>_9wjRcV3aA~qYi<8q%-mn zZ38Q;ZM}ludh9@dOCFueGxII?S4_W=k8-p^7nH3td33$({F`g)Nw;(Mc394&qut|~ z^t8VQ?*|oNEXOsx6xKcTE!-1|^zR8pTH~y(xIc;IWH#DX>V#u1)_dO$)OU>I^5fKY zv)3~V{I!h1?KkBL-RE&wLL+PF+M^j4 zUMzb)ZTQW5?)z|`hwa+FT)pqZbJM3w8W< zI7gzUdEj?C!n2P;yO%zVyzu+-tr!ciUzh~!45|YdPvESwWoIj^R+|q0A7YQ4dp|R~a&b&^IiF{}##td(;Pu2Zb>|oxjjQ+{Yle*e}Y+K{`@v!UF7H6b7 z0$rtJ{Iz|U3#LEDifL-&FJIH=shl%B+k3%1qRGA9Jr=*ifVGpcXvbBK+PD*Q3@5%D z?cLq8eWqvJ0i3tWj6~adhog7znV#_n_5oub2m3q`1*#t7v`5uOuKPm1-NE&qnc5p51h+Cl>z2;@@RiM8au=Nw{owA%6c{{sL;ves7`2y{m&S%yS z{&ukk>tFe{cy}NkANHP6wxXFQ%9(g=J?yOcsqK5*JF&jmFG?M`0~7~aj_iJy<^IAq zbIXe$YTyY+|Gh?q9e5f0R4?P~3&zGrmblcReTbiZ?@|>t&E-1QC)@Q>pQX4*-lg!a z_^zf6$2bLax+$vxu7S6tYIP_a;|0=Oj0s{~)kl()s>gdh-(AyOrFbvpdo8T5If#l8gc@1l;WYv^(p^-lepu+%YiO`T^$_eR&7p=-uNo{W?HH$&$!)kq_Z zs~&dsv~x|1@xTtoRSLUO+C@)`alsA-cE}4A`E_-M^1YSiYX{B-Eb*vAcSNhqWS5Ha z`IaW+I-Rd&EB!r*DY4=19S!c0I~p*)KHwON_PI~f+hbuo7{hW5r$$y^!Mq5++0*WZ zp}$s?;oQikx1I2p&ENa7`n z56;_S&`#U!bAV}|qr)6pfp!96H?nTce;@Ctvv%3{qTPYrp+dCZu1_uZjmh1D>t9!O ztgAdSb(IwT563#zcqs|z1H&a>P`(RlP`;Pq9Jz5v-72ux&HMkj>E-o*+4Ad~t}D3w z-b2s+{&PEHdFPpSSEOTsV`%+?Fqm){>g*}(`8(9IM{1RdipXD@;Eae^ukdYJ?0_?d z4mz`MpOG{h{ouZycH_dw;d6yLBxod5^ zK6@D&&W=<1cs{O&-kpg!8y@a_ElD|F^Q^3$mwI&0-JEBK8Y_+%iKD;gt|Z*OW0lFn z$1)i=`?Vb?H=W7Lc=M`j(!2F&EMp#pwOx1DhRALU!lOJN_q=u6f_jwEMD+3d!oy!1 zQRiAP>JC@&EYLZij}84x@hdPZhsH0s3nsUH%^MS6oU{OEiSHtPY~;%ijtX1$$YW~6 z?YUUz3qxPX_{fiZEFLprieH&A96TOhF!9NW#UtP!>m1L8@+LnTy638lb=1nf|*g)ntvo{qq;c*7||!$TK(Ld($sBH8&(;}Iv+HXDTnz0{ivu{NmGWNHH2mIH-|NXx{ zcE54MeG8VHl&fEN&yfvq>xc3FTeusLx_Jic-7~-3R7>jm^4(f%o?zSWVfyKh^8#1* zy9dR<42KDxk3b*z&Yp2nAJ-s`hD(yRI1_MBLl4i9a^U{)+TG}<|5=_RMYNw1upfgt zlUXa{ThTe6Mik);eIyG@3fZy_C$q5q!t}*-qpe z^XG6#{&bc%damr)x?A`iuMYGEe`h@9#o1NJ*i*0*YrxKUy|;Nx_B=xM*?_$oE8ZWUOuW!-!%6sS zg}+Bm!e2G~-E$KDFs?|y{UrRc{lGY3N;dr8cryOA&W~J#b22B=c>~T6F@Jm~&v$!Y zoC7}*-+I)Ao+sgt`|znJ5qCY()bAwHW6DnhbmdfyWnHz|N}dlZ9oF01LYsyDMCg-3 z?-OdmU2&}r=M%bE+!qK<5b74%W0m&r5;{%jJfX{kUN7`Pp-%~I68amVe-iqk&~~9c zuF>&C33Utg2)$b9Dxn*M)(icy&|eCDOXzz-+l5Br;t)QAg`OjHhR}4Oc|uEtUN5v( z=)*#v68fUhW}$BiZ58^l&~B?GexdO~FBY07v_j}wq4x-VLFjLU{z>RTp+|-GlKeee z=oF!Igqrejz2vjOE!b3`glSQl8hg)Qg0X9GF!uW!B=+K7{R@rbAJ8=6C!L3DxLVsc zwQ8FE)6V^wcuH?~SaL?K7dj@`e|#@(@00)@!8oI5Jt-mXS%R^@Wj#K@*kiPwst{Z& z82g9TQx_8calt*reOrioQwVMe!L5RgylsMwJUBFFrN6iEclOrU$bW`lamJwv0J-{M~|e=~QaGV2uB*Cq4vE3Gts0;yxn;dqQxEV3R*M3~r^z#Fr)5$g?=aJv#*F zh4}Y{;KC4GDmYxyR}tb~6@sr1!PSC|yf=rq*M_*?5#qif#QmNS_qq`La0sps3IBKq zZV16oh2U)=__+|=7=m9AZ1mL>f?o;o-yDKp55X;hO?hn%!3PDK^4Tuflouy9lvp;b z$tfu2!4@{|@q)2WWIZW@hYI!y#{Ql4)P}e>gt#|{xVH%&A>pI?==gDt#CqZdy98$m z9wxX-@MyvHA>o?@V=dZxS_B*YzbP1VIO}N*@qbXTk+1)0T0YYrxda>i#|u79_<01I z^kfS*`l}W^O8hqn#&?$1(-Pv|E_j@{yH5|+SBhYlxR(kZF1SvxTX3`BSi$XrhX{_2 z()>;P>=uk~zO2V1*tDOa?PXesf1hB(zgDm*U-g2Gewqa1Y^e2w^3Rg;Y}&Wb_9?V} zhqeb*A>l*If2+iQro_Kw$fXgfLfQ`kHB^ADW zdJu~xOP1x7SJ?3?Z*E0NnXv)iLIaj8@#ZYe*POxkn*3!IOD6i|(Rfk_p6th^6=l<= zK%O#hc|}=%ZiV4fS)5atzr5I+XIU6mCd^gry+(p7SDv@fTj5m~z~aISm6|bwds0eW z0E-xyE3eW%OG>?EIgpVK(Fk-sUSi=s4kiv}0?a6w3lV4r7>vgI^)Oe%WUpFQk+ZbWI|gC-WUpEp zf~ny#D!XEJskdaA@usp1OW@!tugIy$&vh-Y%qh#W+_I^(;^hH@8@oJ z_PcBGyW&DV`DGQA&_rH-d1+xzuD8frT;VFpDK+Wz`dlXvMe9A_FQ6+|K&0$|`*B6e z&Mhfj-Idaq?#jFhSDCk>vaHz3rtH}%Dmy(hGu4s;?mA!nr6L%2&Y0!jlxv+KUISHj zNl|gWi)E-hpQR!@pd?*W?3qgWLfx;tD#%B6QBHBra&O*=lp5-8M#fL=RjE};D& ze@6cydbaz9|Gd@3IYp=%`NjDa`Di(=^_IDom6W+mxiIcI72c}+3S05F`vM<-;W=(} zet#z=9Mj!Fk0;L))2& zcEvTzTb^5%Um9p0lq-qNMsi_JdAZHil~du;U5VX2v8*g-wdhrXq~xQjm*x6a+X7oY zT*)OxrEH;Hmn>a@9>g^d?U*gz?406~;?+eZmE|tvpSQdeRa#hE@}jw|HhrgMA%Dzx ziEFxc`+wPcjvjZoX6gB8+r^qLEG{T6xu)3Vt;+S5R^*oyi$GOf_+N@_wfp5-y&0=8 z=vlyKE=PP{2>uSRlMeFS8G^qXf;WWVjUo815PWwCeh$COV2=+QM?pAJ>p<*7paQ!2TnY&P15w6c<1Z+Us={3u^SfkWY}e+PLJ@`BeV8bfgH543xe&{}c# z)4R2M{5_iPc}ml$r!_^?))VE`xUoXqD>ZF+NYi$q@f$QY{8R4G*zhsbxcjm2H#kAC z@mD8ur-jH*-3)-C&S{ueq%6bu4o*?`ePB30VcF;j!!pA%MM^tTu7A*tb-}oiFT;|u z{*y1xOIdat?{kb!L;DyQ`kg8`QE;;0G*E^~7yAsMi$KZ$au~+VGC@8?FfN!sEL%DtfL-*gq(EV~K zkYmS7$#)X%lb2b`=H2rfX`j#G23^wCPYXDX)8{JwTjv`T2T&|0Azgw_dd5ZWZP zMX2H5D)^vKHB|H})GahYXqwPGp@l-Lgl-VpD6~arMrE;@k6t#j(yJC>u9)eoRCCJm zRYp#Qf?HnBYDIU8GE3mEoB)1{^_$Kpge$Wg01@8rAABmy1YMj{tmb%^s`)u(DzUT- zbhWw|@n2k7s1hres|;_cx&#y01tqK0EN`yG7-2A1Tg`A~)(dPAKGz2d9{Frvz{5+= z-~(x4U%9-pTn2F9S>Y{O>ScSw2mDKj5Qp8G@%eG$eDseneCE%eke4@MwZ)e>56^hl zd^|3`tj0rYm6CE^QPFwq@${3SDiy~r^U;owW&%^lMfjU%VIORuE3}09pc!b7D6$Th z2c-Gx@Bys(l!J%%q4^bqYrqa{5Fh!Mh1i<-OlT7?eFp3p{$lW_+;hNrDd>D)>M#*4 zI4LnDyjapn%DhMfT@IRoaBS-@!6lbV9ZwlTGaj9GSWCRQ zNR57U{P|W~w9)Zf*>8E!asGE1>VL&ZsiQi7k8|ur*A8elp zDfy%c<@lhtv`6`=^y@R+cQ6Y_POVO^3%rc|MlN5{OrY-ns)sB-T&9@b>?_^T$8E+q(C?ef!`4^IzIN`0&6-fBoD49z68%C!Zev?CW_og#Vu5+ta$o+6g2JNWlG2rBUMJcHl&z(1a!6j*zre|a>ylhd{ z`9R-w@&HBJ*+FI0^Zd!ZYIwg`r@w$VZRH^@KQe4c^(mnU#e&^o$%Ad13P5#|yVYT<8}k zM#1&?;g=y?p)K4bgq?!OzfQPGClRg? zrj8o2#}Mz2;Y8`=XpRzd{Kzqq57-O2^Q`;`=$dKgb0U7^!4YjSMuA2m=9iJ^#Nk&X zEXR8$G&N>IpG^F#Avxn;hIH9^Xsw?peq92~kO~vlF8zt)ud=kL%PcwQ{0^j`^Y|HG zpv)Hoam32esi_lrNX<2!$kaJMhRp>J=0pWXlyTrjz36f8GPqF3tW%twu;vAFI-qOj zw^vocUugXdmM{-|SmH~N=S%Uo>J<6uJkTko%+zLZEehlbOX?~qm6M@u^AuRxvcKCpIGLd zlV9jfEX^-ZT#%7!t?5OH(1)FmhErPj*$!CvXu4VFR70k6lK5c@z zmdNLzV6Oe~X&213KkG@*@^a0Pk5lkz8mcJ4W}hTlFm=brC75fCeB6T1&``w-rcU`J z2=1?;@(4EP4pIbj&67`>V6OS{$r3zBLzOMqoICLeHs>x%1#_*HPnBTohgeUw;IjqS z3O46%HVAf!d!67I!S#ZN3vL~*^*KUtgSd|ryiM>Z!Ht4>W`IwV;4vDiX2D$Z<{0~J5%)B~7YWW1oGLh5@Wq0Cg69e@6}&)jmEcPRR|`%P zTr2o8!5ajd_3t{txR=Iy>IIvXm@C=PU%xJqze!OA`f=3H(5Ijb3qhNd!Wj)P; zV+HRKJWg<%;PHan1;+`FnxWG_L9k2kB*F25rwH~4o+>y^@HD~Mg3lFPDmX!KwP0Mj zVm%uKpC`Cp@cDwb3BFKpli-t+)c1s>T^%QS>hfc*eAG`;3~n9f@=jw39b{| zPjG`^UaZHbQSbo4&4LFD-Xr)d!3PD87p!LL^q(&nA7Xhpmb*}}TX3@A1i=M@(*!$Y zK$R`Ho8VHxJq1?_ju5;-a4*62f+GcQ6FgXOli>3O?-5)exJ|G_259Yqy9suBbozP< zjuspt*e$r1-~_>uf>Q(!7Mvyce8E1!4jJH83GODiR&YFSy$H7repvkJtX|jeo)0jDNvR#{U%Uzs2|$+-m#_K4|*K1rac(CAY zg7vNG9nreSs_v5n+Gjl#Zndj?W zR|Qw?kLx3Rcpiig&wcPI7QV%TO9hvTf3BMG;k=&@*Uk8p%X(}%E>h5#yPgW6d_uK9 zuFdetM~e7xFOpBWKRsHmLTlHQ=V&xuCUUNl^!c!^!zTwC1ibReaXn;?FgyX6h zAD@Khx+|Xo@n0bP%CXwSr&9PjV5`B3E7hjuR#`{j~8>bU^^ zxh_Rba1BXoid>i;u6<=-KUqI4A8{&4nfVXPNt~MP-v?#+;Q1XhA7MF(!)~*g|8Sj# zc((Ad&N~UBXr(J=9E+ENzrN^m83FmYpTgL( z#J_G&lKlC?euwRqsc*V{2)Gwp>9@BJ^q*!cC)8gCY=XgR{EXl8Ld%3ZPGkTj1+4*R_%=VW*-EXJ(>yy?~qF-*Ur$m3drS%l3k6KU3{`yyL zl@py#yBu2nKst?FX`*2**F1l{(Q;+@<50p0@f3m;5)A0x7<=!7lZzBHM+qZJ;N>C1yvs#Y% zemS-NQ~l*o$CKuF*YV7k+-1AXa@Kh~YynIO)$wHd%a4vHP@i->v;66|^9|0|BC90` zmj4peB>iZ<7uzuPaG5_nOh0W*$v@kJZ7XH=p^S5sxbG1Qm*BSr#|wT&ut#vc;55PC6`U>jw}MLr z|4uM&PPW!jb_?Dhc#GhA!P^CI6a0$cCc*CrZV_zu30eieF75{fza&^?==hrjn|+Y6 zf}_O!A;E6J1%l1KO^jfVxaS&o*++8-P7`-C4$KyOgSeLp{;6QI4`}uYYsKBvAF~hG zP5jr1yBXg!2wo@dW}mRT;6`yb?X=l9H2aFp;=Wn@?-9IFaGPM$F1HJQR@|MLTE9OK zZ1y?KcrjYs*NeN^r|c=%E$(KY#_U@<#XUjX9}}D+_}7B71V1I%CwQmeD#3RNt`+=W zg6jnTLU4oN=L9zjeoJt(;1u;CfXu*FJ>=yic z!3lz26`UgYdBItNcL??gep7Ij;Kv2m3VvE}o#1x`HweC4aHHT>!OepITksyizY*Lf zc(34g!T%%Jd6`cCp9Dt>{)1q*;9m+h`=lcTCy2XOu-Vr%`=Tl0UMTKn-*mL#EOGY< zHv8he1pCB&p5Q2HZ_g22CGO>dmF%PT7F;Xt_Xw^N{Dj~J!9Nt-DA@Gx&4Pa-?t28k zAlU3fntj(cajzBkERpwI!R_L1_92}jZ=|?87wP;k^NeT-KSA81#l1qXPr^qFc8mLu z1SbeK5jC+)c0T7Wawb-Yo7_f=k8yJi&X!eXZbliNCMl zI`KbKaGUtQO>n#53=>}bPZI3R()p`b$*^xE{6~pi(?s5Bf}NM^_-+)OBH{ZB zju!XJOn7lWQ?Ogy7YR-fTqih1@EwA)1aA`T6Wn0jh5rD-RpOo{I9uEY3a%A*y{drk z7pLp=%nQ^b9V!4lt8!CB&dy=BaGT)w1h)&`AlSKB>-$l` z(Sm<0*e!Ug-~_>Q1g8it6|C21c@kScdcK>3d8-{S#VW5I=gF>znQwb#cgkS=*k+Lb z6|$>h+zVv)#Na}#_S^XvVkORwx#nTVC0K>9WT=mqe zg1NT!Ky9P{7Q)?d;J#EGq&NKaK8)F|0`+ago3L z>3u-{)sNmEHm48tc?5g;(fdW_lmguuzuiCi2igO@pBIov?>C!M4ZN3w<$bANUmBa! z4hsF9-Vrh_hQX%B8*FN%-mf;NB=k86{gs4HpWg2a$fNgj0`*7ZKzcOhUlX4DpysrN z*U!C<>PHEJZHJ2e`>+)w9MFD5=yfB5wp&opQkz0c9H zKh*Z+(0gb8wY|On)cblScBYr*)ksUseP4d{V&tI=^r3rhy5(dPw9{r$HtJ^HU*KX!Y2diDPPDoCLpy$>Id zNADZ3J<%;p0P7lLunDiR&o7V0{DMb68khUq2aPNI{iojd55zCQtoxG8yP*!eq?I~U zsqKEHWb6oWuWTW9g1ENOo}xj$}i3GH1_UoWbl~Jio-E zv^S5nIc2~%Nvm~v4rQ4YrtPzZhPo?OmjB?#CvunC{PmVEwi?*QIb}uDrYug!9lW?cQrn$VnkSRSod`?Ary1s=|r zPWE*1XX1O#-GWCJS=}9egFrow4IRr5>_$$SsP-R1dF)~AILmUVEGYdPT-8*OGjb{{ zfT1VQbK>K3m!X7L0XvD4(=$hAB`4xlRfpKovNqW63i{g+WS44%tqZZEWqpWUGVB^) z7Y(Z8O{JA;3ifBk(Hvx#X2sD0JE%y)re!f~Yp}m8dTa}}v-lkhvdgsiwFlc-{#3Zb zy1@7xTx?a517;{!_`Pza@knck#5N%hbzJPOz4qF?rED}>yrR>xirkT@iB=qYg6yua z;%K$oWomzIL3Y~TLE53QIj2g?(!SDx`)UjxX-%iEFLPLTF-u-sLxM)wSvuDFKN&$( z3+!xVVhZi}`zC2DZ`$89+VMuw_6XQX@>wU-U~OR}pLN9@z*hf=-=R=$WXw5B<&La+ zvn~dEa*!55z{LGO@?cZh^f6bZ#s^` za{+$k<0|5tTGoz)Me7KCw_wF!r5?t@H0u`W9+);8js$PkpBjXjs!GB8=zfMpGSz-8 z_&$t9+5R7E+Q;TM8t?tA_>b<7g)Aei_u&*}AKs(Rv|;daLUi;so~Y9-do445<|7I} zF@iMcaOWTmR+?_n?nVyZ7~HK5lL9$NXCd5qlv#V**VEEZwF=uGw%^2s_~Q{*_TW+DbQmCM61K z^rvTtDvjE^UZ>5ZC2Ks|BABdCG#!U{Ml&t&c82I_y3l??Q-rQoX|7`OzioHVT~?CZ zR=nhmYOLP52tP~>&!2v(X+7{$z|n`{h{u_VdnI8Ej$Br5;1vcUk^|4 zdw2KRS-aD!hwCX6^S2K9dmqBpg5Tba!QsZ@tN0j{t5+j;-oDe>d~0*Jq@MeO0PiLB zI8j%u@`9|k%Km7Z-8$$;*N;d)hJ*4LD3#KPlJD!p}w{>nX>=F^s%s$gCh z|7Wm$T6c6m(_9T%AJ3(9j5nY^6ZR-Jr<~b=xH|JQ5AInuKc}fEr&Y)Qtjk&5GW7j0 z)Gldv<#IYuMotH7`WuCO97erJ-kk?*l}TxX&qv;!W#wI|?McfWz2_}DOf%dknEpfBgl5Udmi1`!?!dTQ$`RZ49*CFaG>>w6n#Snd z*thQ%@@)x2FR0rB#(Sxym*BFQa0|xbw+8f!J|f!c+d65>_TQ(J3x^S0pCM8_@pw+N!gWZuH0gGGutX{shLAFewr*eyi1WCH>tJoOhkY z-~2<=X_!(8`w;qsp*n1bz9$T6azBHy1x%gz&-*d@Y8ZnDbW*llKf-Tsz?d>Rxf4H+ zgU6I-(H{uk3+QKka9&iw-TfTOsrU&;+uk2Cb)_%VLHtZ5^>uVV&iOjoSK0gaSC6CX zIFw^c4>f^0J)sl1kNyg6S2ObO?_iFn^*B7Upj)eM+|;qZW!qEu2JWVTnJ2uaIy#nx zc$D?*#SZll%4_pZ-F`1RPMkT%ahqnPCs@aMzeoLnsSp{^Z@W^d!;n7Rjk0LzR@y{U z7F*xJeK#-=YlzvdlZFZHBlI+(X9(?!euDn8S;+82 zihWN^b7+rl$)ZF2ZMp6@CyRZeP=<|~#lL8uFs})HAcuo-XyRrum7V|Kc6DG5vWE}vaoSv1$X zdN$l@h%YJi77Koa7;D#;uYygYy43PP@c>ogtjiM9 zQcZjX-7c;4maR_nmfU-UhgoVXEOIvxf%`ciD6l4Bh#*tBx zgP&t6O)XwlvPj)coB1Vqm4%|=MM@p*mR?b~u=pBm*WmuPtH{xoOV2vy6qa*q0L|j_ zdxgGpf%9v$Wl>4Y%fqj#mE$Z#IK(W~*-&V`TdG!fI_4nTtQw>4 zvEJw8<91(4X>oeOvRi< zes|O&gq~fDUkNKI=1+s^s{_lC?w2jybmmmWQkhZV&mA9mFiHHaB=s4QW^gr+r{T95M0b7Qqd8{mJ(V{bp^(#{?J)_ke0JVKv8 zN&kuB|0nSO+s*qo4_5DQZrd!2*tS^w{BL~J3UO(7r7nH@>bG5L$=fU5o@Dc>Uu)jj z^QYD$7_Z%^L$;o!&x^3Fg1&{xPC(IRG z{=TUw$d_Y?Zy)IQ1W#DV6E2U@9m@xw$~t4CX}x?0WNgownRpp~D|Ctfw?aMm{cev( zb>=VB4|#Ovk3TWJBp~8OwYCfSkX~=VWk5x*MPj9LD;JSRfD+)-lSZhqd-$Y zXTiRGh_=TDiW;PThy$u=*iUE4u=o)QuT93!W5*#hw?KG`GsYzYBX8mqL%Fpc}x?H%F@vWd6k&fym z%;zSx2h;<1cQ*E{K&wE%=a>@~y%g?_Uxk(CLNBmC2#US4F#N7A_%tc}&aRb@ji9)l z(1G9mwaPPokJpmB7Ss#-gP<&zu4UAJlPU$Rr97Ywpi#?FPeHRmS0bG)pf$+vG#}2E z!M+W&8MGyzOe1m`ySAVh{su^X#(g_93rd*tp$xL zMtVVgpn0HsKz{*#_1M^(3O;*4|B7@rm$F<#|0|XH2=OG8Va*t{9rUj#|83=HKM)?j zbBy$(-c{;)o(8%d?#@+Ot_`5Cz+Vbhw%!Ddz838tXzleVH=yntQLgD;qtp}B7wAs) z^RVW%(0dW$y9sB|K%;I!zJoU0s?;+oF)V8x>q(-c74#1JyA6BSptb9q~`Hp@B@uA(R+oaSM=&QC~sgdxXvRSELgKzX>D8G)`VWp3=JfZ%A?t=Y>Cy;;e z=X{d#COK+BeZaL1lrPZ%W>$XGfqn;k-CHSVb6E5bbos*XTf^VWj`*h-?<)@cjPN^g&?ZpUqo{2-3k+QSW3;y==>MKYKM0!g9LuLC zY#ZqF;O~2$?Mq^K!cR~?9L-_bKUL}m*he+$dgBAN+9$!$+cho)EkeGu{+F)TuK!jl z4gBL@!ubYJcN6j(^dRVV^dAj7(4GOeyu$YDSK-ya(D~jD`V{Q9{Zi}2`75p8J)ljn zuWP0rnjD^;XgBa}P4sVYz6iAHx0>#0K|2GSy&L%n+VlqWfN-AQBR=}yLwzMWsz7gn z9#L43UppLWpwGcQ{(soNp*??x{e{Po4f;Cl)gLLx%Z?V%O6aHRPdEz!`?OY_k8Pk| zpd7aCg?`b0SHDNrO^&R6w1-~yL*KBkeP7o*o`GZk$uk8T5sv3*_Q0KIww-9l>p`PP z+purVcKrj4UqD?SqJ4qC?7yKs2UQ2rZ-Tah9t7Qg2zo~U$@8QQ>`y`C(U0*gE@P{@~O zNa&yEL`Zq2fRyJd=#OU`NO?wql;;W3n2vwx(@ljVzIarX(?HQ=bJ(Y<@jn{0&U(d* z|LLwM;WpTZfuaajGkhy(H_&#_?x0an8;I?3pl+dE?8Kh@JYCplbz)DZEQZ04wy5sH zenS`b^%zXN3;UK%?D@MYt+3~rZ6B2*Pu5RR_9F~}pO?@kQ|a>5f?2~TgOE3$ zp)5}g;OOyTxZg7H_<$ps3G!4;2vfB-dEBrkdlw8!A^t+&HhCbM1*v-SfbN%5nmYv10e=Aw%{&Gdl=tq1vj(UXew7b4$oSSmk<>I?tfAIGs`lrcV zJ#x*}Wy9APOt;XmorI^%pToRk!t-71(E4|LLgHum=Av6i56oRXBJHZOm^8lA4yM!a zEI&qn@MFneyl%|X3$llFPrZClcfN}be)M3))bWoOO~2v0OD0`)?WM67E}u1$<+m7l zwu6$TEgkCDUlDjAZo}_y3w>w1Yp<9v^?SZ)tUvWIrvAKiOV&B8;?^TPg7S~~V%(2M zxog*scB2|*?P{eSv6+X(-uiU<2)Gyndo(Jl24NWq`wV8XX=k!+?HsKCPdsZf$M?B= z>Ev;!8*?`oB&6NDbaKD1BKVCN<02XU$>Q_oBA-^k+zzu9W+%)CFrTBH_yY8(n1i5c zoyD=G^t>dv-3H?ikMeeu;f;aY-(lW>c@E|tn99cr6BeSqo%+4y)20Ex0(0oT+$sI- zalr2OAFP}~89sq|6UHAN<#b`4cgj@Q{SxM9_Hd!^4=?rczT$qq$!rhxYXhyfzn3 zxAV_poYk4*Z9Q?v7*0lh^6y0ce>#7rsX8<6xbpue=;zkUCv>KtS@ zwe0+nkp25Ge-&fpvq?_~{T1tsJ%7pX+M01a(Po#zyb5zz4BtyvuZ`_lUzxzMK@0ht zd?xMW{a3IHDxb@47&#Ptc1hd+1>XOR_dkR3on}}0&F-O(A`bBXerWpac&oc3-r-g zvB6Twtz$M~tS}ROH1)HQc67u07wG>_$p4y4g4;KQGy0+Ymy+M5xkDo_zC!=<7wy28 zZq(rk_4Cm3X%nsTS&)!UUInjG-ZJ{K!kb532D>9R`<5$;T|E)j4dd_M z@E+PfehRxQ$=f^_pRsk5C>_%Y|7 zBw;%A7l{9t694qB;tRF^CgaB(m{Up4*TFFCKv`UhTBsDnd7RNMh>DLa1%{Bq87!nVFsKf0~%Q2$Lt7_&~m zxanS)w=kCayJh$eJGvQv;GGZik51z_o@`yjkFlW>aok_BZp=|L?(9g%9JT!1GR8?c zM20VnOedmG6nC!w!kewQU+fs4n`@8$^+)3l8D1<2$Z*_wn_Z@^+FzY+Jr*}{keaY| zJM+#kbZ2=m{!T^$x7hNs=m{*L(HGJZ~mb{aqGnBfV2pTewRoui%M$#TTWal4`KDy#uPFDTTikeYu z`#ZG1u=^cNhq%{`d68|lpMkt!4Z5rO5$9h&g0aeXXgJ~(Yb_b>56ugE+;oFH_p`6+ zjK?=&ce4HOHxU?>)laymJqc{XpzLLtgOZy3`n$Q2kN9{g}74OW$VXvG!I^Aae&l z;PoNQ_^+A<5u0q437?HgE~Zf91HMSXOrI=8+PR#emOof`9<0e)0uoT&~^-ket6Wb z!-Ft4+W4HB{H0b|GBW+JLu6tB-LsriUh0khAA#v`UaG6`c6)@k;Tn?o$6~ z3!gFi$NMSmiTpeCk2r1e&-gzlKl11qlTW9r|9?yQ`9H4zZ_7`hW%{=K1nJ_W`mb-x zPms?3_5GJ_e?CIly#eFT(*x3+_zhuBL;e-QQ0^N+IrcVVYQEFXGGF8UafJOs<{Dl1 zzhyIW;E%_2zt)4W17V&pavRFHzma!6hUilMP@%CV?ZUd;Dwsx?_h7hQO~ZHE!ELlF zG#q4I4r7gDDfc&QPAEUteuqJ}Ct&>Np}5`%Y@L}hYmdZOd!(Ny7{;Cs=sbgIpzdM* zCFO_lW1aGH$aBz=EpXn-I?s0kyMqjCPXpvAs^&K)FxBo&NWO+YPneodHzufOSUYP=~Kz%^`{Cw?eKa?NFv;!UV z4_Th1Jfi0gVVFKf<#yM3k=uPoV$zI+LIM zQi#1@|F92gf9RKaNdNXQu*28Er~VvQe(aUoqPn<2Zkw@`pEu>U~(YJOjF zAHS=#o}l~(l^?|a7a0FKd(kapn1*^7yZkrOj&3GS=sym2f3fE!@ISx&+&bo`_BiFe z4ROZ>WV7;t+nLq_w|Jv>TF@`@v19oGZyn;){^C6T0yBQV93bdhNyJ?VV=q7E{R7D4 zm$@qm=m#da+ywYlb`|cc*rWVh17rH%2kc?3F@9I=2jyBfW}@A{{he-Y-?3(V-2Dr<$H4FVM*n#4ur3Sz+uNDo_J7@&190zf_LO#z{Qza} zRDSR@vLnw2BHm8-abVxo{t@?o*1-Hf_TB?3isjoIZHAnaN`{e)3Je(}4N(v!s(^yz zBty=bK}A3^f@B65aux*)z<5A{L_v`tK~Zv)Ea?@T zy=(9K?b=mcv1=ZjzsEtwOWb71fAjtAg1E4Q^S`UVErGca62Iv|`6L18m%#bk*8b_9 zv<29I(*yZ`Z_EF-ga4=Q1{j}E{LS;9E{F@LKlA^W_2-lLq8{={f>Y|``aje8pJ{PA z7vx{{=My;~{+!^v0?w26OgK1C#&`dxyvT@#5)C?Hv!FmxyJLK{7%#1zqh4-+RAVL ziT(YX-v7z_f0rMW8z=3V0-)>vv|jV4o&4E;{EMdGB+UP=`2A1iob-(V9RJaO@*nel z|J(kP|0)0f6?A{jQ~Et$>EB%l061A!9z05~DQ1@oE|j~|`5 zfv4_N$ibiP@t^M3|K%?3RQ@cORCN-MBgpsE-Gx|B-8G2nPx%n>sf%O8KXpSH@tRNK z3Eblo7c8d#AGt6_^5$M=v8RXcN4ddCY0jx!JGWi^z#O0G&P0rj z)tzUzwEAX`*1J0rr3A(D6Xa%lw4$@u&c6|>phD;uLdOQww}vEwl^0n{)wDsRFp(Jb#7oqEnDuX>Jj+|LsF^K@4; z1eKOLRWFlE(@URq3)8(Rgc6y!zeoQ~vpc`_NIs#i;2{o@Ty|N?b=z`W@>_ZcCqkd^ zi9Z{vw_$7G=PsD`P3@+VvnE@(IoKI(*ot9GUMdn^9B z6ku4vk39!{zb1 z)kIHOTf#hiTxvr>O!%j&^|mNJl`1&Gnvo8sd&Br`s>kHjPZ@CqzTZ1Oj?*JN;uT;f zTn<;WdzKjL>S2>_n$E&i9QsV z!}`vDP`IHqfMz5vs^Y{S7c<) zLdhwJqp(3npP9&#i=<=ZT=N-;u{}RtJYuOci(HP=^yj9<8+_v}HkKlfv{Nx?>LF(> zIP;_Q>o<}7w4<*+BKIb~Yz77k?3zX8%{6mh@A%~P=wf2ql>y@Di%M}Co+;66*FSY` z*pILIrwWjKe~`HesrP2LJuegHVK$CRe0{W@7|cE2YGSf#-?HryJG~^6TY}7as`0A;Ydq34^dSn{slkEsKkIOo?nir%WozmO zVD>_jGla36&n`gp!U~TU9`{`({KbksG9GBwywEe_MTVzYK&KT-w$`F7lp&gsAXfBj zr{G0HzStx$Q7r74Bth!)U0CV4ZmpGHd91r=tdCvUp>o@{DehCMt<`AVQ0@9q&coJR zRs)&Fy1ptW$Ks-jT!lwN-^0sEaNl_tbiLdKOVk4BZI1QzJnu=4QgKVi<~-l^$n_5$ zs@OukvGJopQxs?3BEQON%D`_6{eU%@Zd(e%7)ujOo zHk@|?wggFuAm=B(3E7kwE)jV%dt2O9`xxojAvsRu`CMYXtxTsT=+jkZ$*T9iCQ}8N zNO9PAksC$KX%CIgi%YSD6H>)2e-0%FUlOahvoPZ8Xc2{c^*DG=R@yWeD&^!x&YZR!z$}8kLJAE3O{&= z`w&w#bgkh#sx2!>U$lqVfQOyQxz{B?rPQ^(J8*1(BigLin19&v+Tx7AH+C z*rh3A?+*b3W%sqjR43WqG!HM~E0P6H&Ui-_dLCo)8zTk#zO`DfunyTTBpVSYI}#Ck(5@-v)?T^w~l@5r%7)gvoO z@k;ZMF7#VXe#st(p`Q3NiVO|QG;9~xZCnU(mpt>4%-VRiCMV3}r}}2F%ac91yt8~_ zq*V&8Vq=E#qQ&8x#?Xz+VN4v2Vb4lF%@@2mzBF&}QcQ54u{lbHdYGVpWT35Sobt)_ zwAA2E&(;>@_U}C%c;@?N=w|%E4+Cv&cUCY#V=PSduP5I+DqWTL_E-}q4tnevdOzTn ze%^R3vi5Du2358Vs> z^k;0_LB@HvqnP?jc9WGvwt=2*KD4PRbZCp`w?EVc@l<~aAF3D|S=-8?DlS+URb%z| z;C1DdUs0t7_IqP#?v+ismq}Xbow`i-shuu6h4gM8CtI^@4}D($_Kbyy2RY0~%q|na zET-VE=5EeW#qV!YEc!IuhJH*q=?2>L`U5^ZLD~b4c(N{vJe@i)RvUOt&|H@miphHwPM`7x zspsIQT{Tt)-7pK3RO9Lzcx0BinUZE{BKGy6e3s(%rzEI(I&{U9hrx(X zOPKY`7fl43pPk?Tn6(-BvRtR4nWv;W5J^Wpmi+)@`R!6{D?Ltb{d=N=bjWZy|>Socw0R-L%nm3bm?{m zi=cufwfXyRhJs?Tn91ib8l)YKD_@>|T4dFSK;5j+bc;&dk{~fwEzpeLZVSKm;qolXjNgF|0iRvHcTo`5 z{)wA2G~J*D{TQ1o42u{H%^{m$6se4w@yggYF|4l0%`Tna5GX4pGNLr0jPA**L93jf z=QfX-($RAx z3cidOt}9?4(^z^QgkH9LX78^}bD`$YhMQaD>%r0T%V%F}#s-_cq5DtM=%nM1ao6w| zeY#5yFX#_2P`xUN+k?O;l{{7x~DYO|N(>orc|OZCM`f{!7Qj*uh%w7p&&{ z0%I=3-bDr$Qj_Uyv#Z)SZvF8*&ZM`agkDqdJ}cLJu%?PXhsszSzFnBZ$nzLVDdEM9 zm!DodmR|X`IQVFic;YEO{f6eFm8!;6X)l%%T0K$wC<9QFtBW!&!LmdWciTBao@Cib%cD4aRs^m!Ol^K&aZyxwviR$r+d=BOEVl&YDPI37YUDr9Q5mshNZV}($$a6yyOVHaB+($fCRUs6C9f8b`#!sec*lcojzK(FMIpL zO{)4_;j!Dd>b|%e=FIHXD&4sK$H(SCDK0)kJ|EHmPwthNsHi^XGx$WI%WA{TH9ezM zf%Sbgp#$$DGu@Ya=39heAvqUWs5y-ln~$j`#@|{rF2^*Eza7kWUak5vHd@^{HvN_D zjp7aMDw~;kt&+@y8yEH6dphY9lRdrFV~VN|`CVhL-}^Mx)0jQTVhTgtn?M!b4KmKN zoDov{AS>o9Pdt27!DbdU!Xj%sIdQ8lcxP5-ahoF+RO$sd0vT)udFj+>>GH92?3%L_ zrRRfM-44iGUyHn1pR4^ix$#;dM|L&qIAgs%lWgrAJU|Sdc3FD7)OB=xleA% zb@E%rZfi}V-Ya)xX7Uyp^=>D0=e;G#<vM9I z^&DheUTwNCiETXKzKdeedSx)clUC=nZ#z|0<7Y8KDUQ0Er56zP-PyZaG3XN8-f4SK5Z1d$!rRpjW8q08tW>#AG<7!cl@Ad zhT}`i#3#pn&$~dNoyqa|b11NBY$w0PmfV*O-*xRqfa%g|BDE9o>cw^wPx((T$PF6r z#i@U=qT3L47~HP(J zxObb|m_nUk2!Bax(o$SM!NJRQ;Rm9(W1r$z)&$*Goh{z;_;m@&a_U(M+|q38ktdvD zabr~sGRr zygfv_Dy;h%df3GP+0}^D$FzM#QkOd)d;^C0tYrSGbd%^6*>Gx-Y2r{H{0f9;mSRtx zEIUEw3h8rb^?A@!?Bkz~#y){OuFW(*YKuN>>YL{Dj_!>nJ&}{ly(0z^tFfkW-}Yle%YK=O+~^I7_Yc1_3krG+pG^ z?mx=x^F7Gpajv#qfS4**ar#{?Zi$3TJBPLC`eM2%N64i-=tBOk#tnsl%kYT2hl$~> zQNq&!=+-^?z|0eHx2k8U%W&{PGL!6I89!sOHBosFp|c)^vtyZI7GoYe&#G*MiBO`W zZJL(h%X&~jKJVuCPcFKcOewy`cLVR3UOeGJO~wQf_7k=EFnn?HDL3yEGyT*$iZC2* zlFgA^ZfDTS^F{G+@!D+@l5NKHZrk2Ho~{~rc=$`VG#8`UGyj#4g@3^!EKa9O0 z4_(f=|1@;eVt0Ix`a48xPb@5QV8~IWb@X}CLU7A*a-{ac5rKE=_1qT?j;DW3nAi1 zFnZF5hErr*y}DsH_d;*sKuk>B7B_s?HZWQB+X0g&F!o*ELZPPkyl<_0XVs1i`7JT4 ziQH$@j)`d0-bDv2W|Rbf#?c)l;-(g`{rpQ}P4B%Y8D{Nr#Lz1lxdcRL`HWZL57M-= zu!)hP2BDr!A$L;_N7r>ISLrn+jTck=B_}Q;XBFqK-w6{#&`r6(Q5RzE@d7i5TufPJl0II)20l=y*6A4@kyphK$GHAEz7uDA9Qp3uyO8T= zg4l93EW0~5O6~?8h29>o3nZirF;Ww~Qmj2@ICk^_S;30<_Q3O;+5B)h1s{G(Jc)>b zeYg4DkK5GIiu@&msda*5Bfg6`cc3cIgQG+8MD6|SdS|@GwAJvVo|KLq%y_Zx{+mXLo7|69|CL(pCN_g^5Fnv};MMzRqXP2C>dJ+i*vgCe4aDtmE_# z>jguK?_5J>wS6A$ML8HM(pD1c$?=jPbLr4C6hrSU+b1QDs^OA_PK&tqgJkkpyMb@y zgR%^yySCW5{E_-sd%kz(De}Q#A_e%b{j`+9}MD{(A z& zxg3%w*Vy}>$KBP|dv>0@WM@IqV>$i(o*f6{f@)ZliJ2r^_e2q#Y5-_3OsDehXC;7^l7r+se9`1Z^b%@{&3Z_8#oJ#;DQ z*OCeYMpuA{$Y$0omt>t%cIC{k*t4KF^&#W%{!*@C{!O9O_9q1lFz&5S6>>tDdA8gF zq;=xM>m>u`CH`n)G0XS9wk83DA36p;s?#;8>wSfFP-8n((HNqe6gir*yQ9=AL= z^5ot4ez7cA>R+FiZ3T~R1=>I&FURipQC!3d0(7iin37k&F zu>#DGkzs8CGZItVT_#JB{#&3oGymSK{*7DdrA4be@3_{a3C(^g^+Z34N6AiIJ>pYbu^&UOT5}{I;Y|VK?M0-AHd4 zgym20l&uzjQ8Pm`s>vryV)|g0u=>WFcQ}$j;c$fbSlA?r$EmNLR^aM`pEubZbv2gh zrJrn*eW4;=9bn;S$U@t}&}lbWCxKcvEmmz?tZ9K);}Fh>yDrMqnE1#Cs9l35 zN>U2Z869P!!B>Kt5Lxo(`&z!A_tfAV+&M3Eog9O*uaka%{2Y=?H@pbjo6-vndd;$q z7!Q_@mq`$5i=(6agN$@65_;~$bivOY9Y}*m;`D+|GE$fZ)y!RU7GpD>5|}~X!IiKx zxx3m`bRMnrLqX#lUY#@P^szGUJ%2aT+psX`4Al#Au zXrCWr7^Jhv#M!Yp^TB6A(jmF@%cd_ucXD9OH`YhUl<8I5rWTz_#QNKCJN zMSawJS(ELa@!bki3fS%N^K}kX@}XgTF#BJF%!y=;a`pj^*-m8NNyhFxWGAp6dmhs} zZgJK;mmY5^=qUNS#sbuoaU}ru^2h54s?fGgJ9d71&d_Zq7woMEOlwZaOv5smaXSrP zP6giJ?o!vDqyU0oZuFt;=ls5+Lj0}UOS=r{QU z{@n4}NikWt1vc__x8!^6gYB82{zK}`DNIH2j@&O+ULw*d4&q5H@|MUkKhOT9Rlm)V zz&rGemaN>+vezB^+b@I~T{iE>PQNA|D@J;21bI^{54?8t-Q1JAbB6IApn|q`4pKHn zEwn<&sXWwMsOtH0*vxA;a{1iGn&ie#Xx!~a!4RIf+!klDOI94#5as-2_w`#zA%rW+ z`OKTJo~EqV`x4v26`37wF3nBt9>SMipYJ+H6b^edp6sU%)r~~Eaxm_?i9Ofm-WYlltF{~`nxDIKCe7qzD7+D808*$PorOY8n?tfeZL**i->_}_Gf`JH4i~Xp zMOq~c#oqL51;Don3pNlaX8&SBspR@ zE9!D&Rp0p7lv^nLFV3>%jI5kkg~wxq z?RQputIbT0^9Fvd3H(4&<+F_j_SSqFuH`;fO;z zD0Dnd;mXJp(i_<(k1->zOuhyE@$6zLeQDV)^v|N!@3q*J^o7=e(Wg5T%>vDN1Maco zlUu@gO-hRX57A%yLKT*rx(u;D@sYZ5LAmRrm&Q^AzkwDhYnx+@@R^S+bN0?w(-mF& zrZbBT^=3N8H@$wXagd;!!>n-knztVL6m2I)FCogs4c7L--tgnIMC zt|Y2AsL0at`cCmN|I|K(xill=k+t~5m@sUf&GrS8^k?s5PCfM9nnUjGypI8y?ngfk zy^pNFgqVT=pF>|7OGqM69nw>S1_Tt_pc_{6ft<%7HC@LnU!ps?^{6 z*P04WVx(3a@ulNxtCS#2P}Z9!3hd$+cQO|sV{tH2xu=d{?1=2V6@nhMf3c1MzXxLL zslZ>@R(aN6bf@wdHltJq__(CA3;YAju)Zv40J3}iLO_8;d;q`!)QS_138!7LD zyHf^-r|G(BsXo@%ai$k)wUxAU5ANyG4n^~dQV2A5Ho+*&R zQWFM}OUnk_A`Tam>H2l=u4#L?3$Gjc^Z}y6!n@{>-8zF9PcDxD8_MC(~ zop>zfoanN5AlKYR;%z6Zj2T0z>MgzutwtL_B*| zvq5nK2S5FJ&4Lwp=u4sc#2XC1>Xxh2^*l$+hebX{Oz%2t<`5M4tMWQKShYv-4Il2} zRCph`9>w7ur$<)gYS?>g^dwR;5ee>>*^Ju`jql30<%C?A?o*VImuXNRkaaSH0+j|F z?%U`n)EpA5=_KBgzS%_1BP;Im+Dc9Rjt|Eldx}t?{L4Q==UA&n(D&niZHr)S2i#HD z*#{JT6gIg7j;hlay87}hS12IvTf^n9OI;D0X&U~E=3^3!;N8=@K6_Tpr{(Ci(qgRX z$H}e5XrE>I?yTd*2WNxPE%8qeRzU%e+S~490)tQVy^1klt^f??=5xGoVQ{S1Ray_isCnK5R-+z5rP~*xe`-< z6dgq$XZ1&(fj}C9caL~IqMyrcNrsLCuQEZLC#P8Wv*E)O;8BeB-jqGRLqc*Rq&*}2PKOLJzg z<%StXhk_JOWAK@?Scc1ZcZ5hj)5&ZQ6J4zjpr2e8xuf|iZkWTT^>k8+Kyz}0epk*a z{tDMNSJX#mh$lgQOkANhUB0h857dmxP3tW!DchS4U#*Y_wBdK5J)U_d;Pm?9S2<(T zr5=4CpHSUf_ux-VG=X8tE|#!`TeOPlM zxW!7i+UJualiC)wn%6QXc(W7+RIgememOEWZq$XlY)X=wZ-`6{QO1q;wzWT{+`Y`S z1zq4BLIix3Fl}dZx{eAAWV*iPl_oxRzp?D|GoP*7c>{Fs?R4_HG})69t16tZZ4bNp&56XNn`C5!91 zsxZDm-z=vJ@rvE`L_EJ^IW!0w~i9g;*;n^5Bn@<(wx_* zLPTt)7L3@8{6rl~7S6>qRAe;@S6HoTEkgWR;@JZw^h`2#mu(FA^d-Gu-mk1MmoY0i zNqwog8`R&XLgWIY)PGub3ZT(JubJ(~UfoZ5kzqgnCM#v@j@S`Poa(2|+p){WEiU3~ z9NZfRXk2KAHU-Wk+8$yT6e=_CeaT@7O8;lxKYX5zvdn3ZEy_9s1o2V!HGSOr(vj{E z%_y5rYjsh8T{b%R9%HEsLz1{J0Db`7u<-Q?kq-*JsYb~ zWK}Ep(JeRqex=|wo;IJrw1k~*Gzseq>nl*JXJiKID9u{u<#s94hRpn-nQebbdlN;nWoJ$2zVBSML$TVJcz=e9UlXakHf3hsao_a-kornu@S&XhQSel&*x8f=km+k2ikv zls7&LF>@P)i$BHFNkDC}Wh%yd5b-2tYG>;`>)z3x(!n*49+CDN%QU9EpMMdZ5kMw< z__OQ2#enXQm5(}-ou2X`dbi3+AbGN|gQwV*l~y^%7Z5x|senFo6;BvHhnv(>&*XMC zp9c*62X~c6XQFM=Cc#W1!5gwfhLrL6Ey>9@o8jdt>%Ychv}%WLofn(lEa;ZAntSt9 z#pbp{hwxQ0f>`zND2M=tmZKqn)9{;zfosBRG@(dLC@GAFUjnY(BliKp?QF1q(h`$#37<(Dr8&n)!8eS9X; zP_`m=zfZjEt}1yXp6p?y?U~9BKU%MKlM%pzxViA`zx))1bEgw#h&gC|y8vl*O9wgd zo`_ONT&Ka*R**u|cA}1&tItijFd(xe58e}#5Tqg5RBnUy-!OZ6S~vmgXo{nt9=64B!};&IowICtUHt>eSd*(y-@2|a*!7vjic=_g0@lqw5U)9 z)_|bGZH*5*D4lro?rwT%FxI841E=YrWgJjU}a?pBKWIge{Cd%&gw6 zRRYGt8t+F_-I&(qw3vv$p_2KYvK!L0gLXzMpR^V)jJ(-ukrH#(?w}?aa}a!02hc4PiKaDRGJz@$&!da}vccfSqQjVh zPQY|*c`7XlYYdPNNh&m55`@eZ(7Tkz=x2a4UA{&~* zI<51QnGwn@RJU2`1}AiN9986x$i^oVg+x`7Sh181eK@DbV zlZy@K(90D#W(<`H-wivHN6+tW^9n5RSEXO``}AE%X)ZR11U(EQbi#GP!KJ)!JBNnE z8Xax5j66pOnNM4FrFtGHM?6?B5@EPFfJNP%Msi=rD}-F=uV!n>Gkb+fu`CcQz9xUr z??0B2=L}28REgUel4jD!KFFEK$C2$ zzdOQssfAyEf%j-vc{{G048o<@t>*sPtrcJqW74D^UN^+f8H_Y(r|_pxou?6S(k3U+ zB`&WoNh;QoudHv>)6+o@a|CE4oQr7VmQIUMt3Q7wLg~yIFmOF@RwjC{L)I>8HX z4&`FIB`@XvZU2bJjeDKCE-u;AudYghrqsWN(oy*%lOsQ75a^efWY z_w+R>I7zS8Ne&Ix@sYHC(3qbi9S?xoZ3j}^@NcyxgqD*-$c@;o^8Fs4wAwjlq=@T_ zWn!MJw8x7-5-cN)xjAdr;AkXUVHt74k!CovJ|~U-Y)7ePbb0!7Gq^@rlEW)<)jKLd z_0!AGm&avDLTuqIH+oWYHbs}o&h#uFMz?UyQ?6EZ%9y)zf`)I-{@MIeI|KO;TFHv3 zE=2PUY^}Rj&-q9E=S<}z9-8WuT|D>UEEfA3*}#CFoyeHaZoSNX^D+v?Xdvsfdd!&}-G zxzm@q*Rn~$%+*kdoypdkr-5N^5-vn#qz&SA$MR8|>n3LV{20a(bzK!H^b`_Z{h902 zS5?NuR8>Q2PQ?^rf5c)YHLr-Q*trCUp4W;6A@dqHzccg8Fod2A&%IiFd@^?oyeO5c zb-wwJ>1h4ZF}JRCl8(|OP%gzg$Ie*r%y0xuL>?1N1xO^?t+c(P&!F@$H@*Ek~_IK zM6xNE#KG3E$78D>-gwA+oHJE%b0EYqk3Ei76xuY_%w^w=SE=G5)i?E{Su}{$_o|aR ztC@Ays#Ji(U|j6T+{fl-9Xk#78nNIQ^FrJ`G^0%Lxfh0k;hqCIoa3A}DbSOvCINTD zhvd_CntqBRFM&>`8V1v+<%@=ip-C5x*6VST8qCSmuVKViR`b=9NmLovO<@VW#nGlk z8eXxDnhBSa!e1@69N}}oX_iZjqrVE#{GhjWJ2)$)d-)lwrp7@QR@i2=a@hqIV$)J; z7MM2fnDf54@r`-}Z_X}NVNT71!~s3i1ec5pQ!R9#wbHY6_F;`=NEdFHMG;-TzWk%% z#Q>7lBf?D2!yz6T?FSaC+fNWn8sUsF^QtAePl@$pyx7~DHw?}s-@;hG9Sz9Kn|n2j zmN3s~cS17(ktj{*#9Vb_$(?&%Sy)v#_VVjH)SON|=l$s_D69C=l2njz+e|S3uSiIV zmJ$8(@`V8>Yl{Y_cAfj$wLL6V^6#{o53aUt&zM33;KeQWqgAV<(o_RdKc4>;B-$bAr&icc1c!Hpk%ec|_DP8zhXI_PG(wVJ0@KNTv!J!7R&cbd)0&mX) zzgfga^bSHad2!YH`VwerhFnQG=-z(8{nlgXONiK38AoSB&v`CroJnf-Ok1z~LolU8 z+08*T{PLQ%%^l@Inp#adhk-bWFk^KwEH>I`s`$qm?EI&ZM>G*Pb_V8MlCrK=X`hXB z7@k<(;UZU^pNl-H5*jd=47|uG;7+`p@=A@PS-?@-O2PW3%z)-$R-jWKzS8k26 z{e-#lUnuIqSN#-y8wsZDsKv`|i)>V6AU(HOw-xWDv{I13+-@YDi@+Z!-{+(;ZLQ=S zc!&#lD8xl9byhZ*dC?OG+H7)%6^rwzu1mX5ZH<7LVY|W&uPZ)4a&N>oTE5n>zgK@= z9lSot$OkPnC`E?Nhw(f~-lH{<64|w6*A-h^{EH z-hwMrbN0*D9G=-8;eOoYJI`P?6Z1mHZi`t7BElsQy5r4GV zUflI;;T;2w2-3SR_uIu{SkB_OQLLl2*9oYbGPIFHeJ=dk&>)f^M#4a)d#w$(!;- zZoXki3Ux%U=M;Rkc_iTEr*a$F?j3vA3v8aFn?1Myr-k`)|+OQ`k_k8D=r{zkT4_RYN}FbIs$ijHO&Q9q)QNIZSVG%%$()))jzr7Q*dbexL+D+s zjlSnAf_B7ah6;IWaPHi8bBlW@@?)so7EJ^V6~2$x;r)}jd_!`sVw4%T ze)Kc}>htXhmNfX8?=+5=%pT*3>1rFUSJWC)HQltoZpgGX3=GM#Wwlv~U+-vw1V+Q? zZS9mU)|s;O!NVMaCEP&n5Vdd+HrE_p)GP~Ff;kP2R^RUdGv;GI#eFY$9l{S9Sh1I% zrSN~^p-ETM$A=%*Qoq6X#ozbJ|3vqiTABmD#>%_#r}&m|h4trGiq`u&Jtr-$u>50~ z8^6^2OO=fyyXyvr{>+k18lXyUY}Sz^SkjtnuZ>kKIZ9$r9+Nzbxpjmi&;6a2C4b7il<38E38tjD*z7%!x zZSIr(s$mcu{-^I`w`zO{f%wbE1J-~26&`-NQ#IkKj{;vDxVIn!$PE0dLHgiI6VidS zfnO8+s)Os3qmf`0Ea1ZfS9qttzg@5-?eri3+)tkV{XoG^s0<)a$Q^QkoFO~F%>hC| zHh{A;WDB_fVvSQ81o+1IpMT~jvkzhA$JV*u5I6v9SO?n^O@=kR!Nq z0jN)CoFNae%RLH6`cn=B-GI)4@W17$0$LBq>NFm%r&K3lP7cG<5Y~W)51@d9*xG>S zUO*-sJaY#UUBRCXxH{Q8_XOi_p0`f#dVui1d5J(mr*Xe@y4%|cM$c0kXTSyZx7h!O z3kYwf^MKColU%INc#@ z;Nyp`fM?ghl{X{?34n07AclYAed835H3;i;8p;tMJ~>XLcmk;&&}B#%{J8`Cw&3&t zcpLz=Fu3}yl@oe7hz#)6`74em8b87JTQ4WRKU3*HLJ5KJzTkwQKXs!E!n%N1dx13Z z{F`RLzJ!01bm>&z|4#c)t^d}!D&UO=5T9t{cTE1n${>jQ)cXI_z=^e>0Og6b{7yRv zsy@vga*DoAD7d$cyN82|vpk=O5P}bG<80+(?ci)D&v)~Nx}Y>4+{4qt+1di-;%p<& z=V#-=r>Ha!JUnmxT^e8ohs$}odwF2Bfa;?4*6a!|6evhe^gZ7on9Hhgg5f3cJk{;vhf3IAL4ce0V-0ADOS4Z=Ua( zADSPZkIApj$L06uPvx)WLs%Lt9E-prvAS4utSdGY3x=bx)mR+1A3KFz!$Ji#1@Hny z0kS~1z`Vez1fGg-Pm?~H+fC^~};f07oWT9@Md7*1zXkmOIrm(sYSJ+=T zRk&6N710#Iix5S~BHbeMBG;nOqWB_AQFRfnsK02cXsrk;rYVLOBZ`s5y2a+juEn9n z@x_?p>SA1RfALiDS}{~YQvxqRlpsrVOUz4LOF~QHOE4wXCAgCQlBtrl5~!4>6kZC^ z;6a#@ z2!Jvb;4D5x+5xZ*1GIAh?=C=00x&ZI)cgRqG(fHaup0sN_5i;x&_D#xK`PKfG0;N; z&_oB&#W2vu9MH!u&TdosMNmHw=|+OwY0djp|qoPxOA>` zx0Ixev5db=x=f?YsLa01w=AM8wXC?Tp{%29xNNR$w~VBmv7EnLx?H2&sNBBXw>+Xe zwY<2zp}eDfxO}dBx16Mcv4X!sxBRksM5aDw=$wKwX(Rfp|YcLxN@#?x00lav5LP+x=N$UsLH;|w<@A4wW_$Pp{k>5 zxN5Fyw+bXW0`K(u03;Yj3_nI1qk%EP*kgP#5tvj=F{T02ff>fkVRkVjxs18|xzf2B zxkkD6xxTp(xv9Wb8i2J7=g#Hs=91(w=JDrA=V{~_<=N-?=0$+qR19)b2gpftK*uEc zj6l26K(j_btG+;^srkkE4f!4U!@$CJ^GSep@dL}!z#9E!SE<0J8n7MMVeA}s7fVvW zSioN(U7%55RA67=TM$u@T2NfjP|#5@TrgL#TR>9CSjb-}U8qrL1nkfk*kCHKzXo7? z!@%x#3rT>*@dInq0G4JCtSkap7=*-w;z9n${U4Q^5v9md-BR;X*V53^_)<)1bt$g2 zzjUf}trRMwDT9|G%8+HcW#(nBWuay9Wtg(+GF(}I*;LtD8B|VF4lhTPBg=Km&C6ZO zL(AjKG3C|exbpt;sq(dQsDh>fUV*4UR_IokSGZP$R>W6eDyl1R75x=c6>AkxB~2x~ z5>bh))U7nHbgc}njIYF0R#)OG`zxm^*D9eZnksk|q6%51TV-D5S`}IqUxlfvuEJII zS4~x|RYAIVpiT)63=Ia3L12&=U5q)#6%&ey$6zqk7#yY_Glf~hK)E!z@LWVLGFLa( zJl8chG&dfUT-CWaP;yP>uH~kJSla{c5hwO-jt5Zy2MH`23O%U>f?5c$7e-(!M!-&r z|0}$SxA$5#WVM!SDz$+K6iihQHmRMv&s;K^THO zSSW-L`A1j+c=g8__E*?oQ=z;dFK<+BKskA=L|6R6y)$}P)2;`bgqScoBqEH^1rbJI zh{1>9!C*8R2T9M(Z$62UEb5Gk9C~u@gy&x|#iIc62@oMikP_kFBp{-PT~QWcK`@ET>tF?V!VPTPy&XW!6v56wDJp`H5S2ij z9=Cy4T0~R~{7cA4$lONU`uE5&BF>#kqo+2ovGR1VbNQDnLBz#VS@hh$pXhp7q8zN? zDlXPQo`an;Xd1xvU0gij`r?v^Fc|0G^#IZZKMY0<(h)`m3xnZ7F|$wTI0?8tg^tg0 z!4jkDDkUG5U-`(Da8cn)&^Hl`o=f?IglYY#7vEAQ`X5{`4YD>i&~FbIdY2CF)_m* zP2y71PgzV4Iq2lW@V#gD6>SHeh&cRmsueGP>+ky|QTL~Hd#NBnSmS@c-Qj5bBA6XhR)&*tS*KLB)i5OzdqY{^nqG3Pe?~J z-c#N2iUhq6p_q=9P=nytW}6}cN5)~dU)>}>og&j{YAVJQE`2X$tZpO5^Rk<8nzyoI zmke=;Y${b#aK3F8mwTrx3lWZ(!>>iQQBq6y*cBT@{Axhh{BmiZWvjXG>7Y>FYA;(e zV<(ngM=}*ksDWJB4zpG--twp>O8$wm$!WfMhc(lXx!qSi(6#glOBh4>qxrf{Dl{u{ z=p2KLSPd%(=|^CnBiMlLu~8A6BRC_vn3#Snk)k2Yk5qPr zOD32wJe)brG;Gv(c*g_;AWw!Ps87i0PyB=keBdDHPj2uKP$0;NNI>C0NJxy2k6<~u z2`4y%phFN55r9hwK|(}iho%692SVlKAqRmR;xa-SBZ(0Y7q$2FbiFJrY~_v;0?k*D z6|F2zE`&ke`&-ugo%4hM9YXxKG(0>4egrRq`?nu~7tZ(>KNl+x*MDR9_k07&5BxAZ zJczD2clEJnda7!JgZhgyZZZmNlwHGRbhBDikUnv742Smlhcfw&cLm|$%`^>_ezFEV z&-aH_n}|kCtn5A_&plENcfT+sdUlKM{vUf^O=*#K+sb_I@3_9fq?xb5zm4JT^r-9b z<^|f~^d##dVK>?jjP_B`%28^eP;xGXfdGfgFjJvU_D(tJf@xUWB^UXJi1&xV-Z2v8!jQIF#J;Hr zR(uy^7-v9R0fCpKQXAev0FelVsT$fY_$+<`OtVd{Vv-nf3wis?B3L@ZmDu;GS@tKz z>6;tZE!SMvEGJkez>gHA@cLCH>5x7xQ_Q=JXCwNOt}H1}X7o~E-bJ$taT`P97jN++L3l^1CfbuV{6hbz*(Hl z)_G?3kq-F`Dz;zh%zxNLr!TI2gJH0OnC=>0C8IXHIaaT0fH6@;y4&jJUlS%@h&n!w zd|~)%%~}Ko|HXKUIVVD7rVmjsMuYhx>~gS|Z`H)B`N%E|Sw}M6eo6OPz++#nl0YZE zUw45FX?X2+XK1GNXNs$Fgf?h5Hu`OovJ=(QDETs}*Uavt+@gH2v&I}9G;-`5lAFAF zfk<4if}qI5-K%F_UeG35tGhkUc@j;O?qG!X=Ep}4+MyX18A07=a%K+xMNEH_vp?$9 zFF6a~eJI7j+AH7Ln$^xOLa=frUxMf_az=2XMW=Fx4xoZ^2J-XIibnoj(Fg(jpNfWt zigKc85J366pveN#fMkYv#^q4abAkr?Q75!N)w3Tn@;ly7=>7xlCu|4NekyF)F znm+0jF#H?`F_iSYK%N`ud?1|aBltQ@1q%-kc3ljKsy48w zCf^<rqhI^WPMwQ8YO_`<8&&>HLZGtS5N&07pUZ8i z0DaE}AA;`T(_sCnWIUZQ95OO_?a9Zok@CK2;o)lg9+r{U7WfNzF0jFakBgk)i;9EK z6BAG|KkZ_6N+rh_2fP*ZfOq*zCFbqNNSLQT?bX&+_Fvi|wHl0bK1w@MfKD%xhf)nT z&<;qHYMPeC)UsUTiPquGtW7C4tkGrdTg!XUl+pO85Qiz8za3ps6dL>C=Pw4yM*i z6BS7Ln93PJ8lWwk=S2liP4F|&&(XI0rXh{Budv*=KidfMAs2^TuX}tnUKMEl#KOiI`06@WL zMaBA2v0O$%gb*Sr%0Yi5;<$vk((bO-4xqp@Gyx&qH zywqoNj4=gUnz@haW1l0C6N{zo7IETWV;XP07n@C!%e%D^lEn@FHwE=%7<}qjmBtws znV_$eNKU{|O zm6^*Jg}7yqfaXh%RQc4c4BuPiP^`Hf5_K;sRS}7Fdldy6sfsBDN%`bn{e!wz}isZLKK*Mmad%&d06MxoW9mJff6ar6e1-VjWCJ@VcrDhaOdS#t<9lzmwy^ ze`7A%+>SL}7Lh{c`k}P_*`vwW0r8@3@gpqXdsco0dSnF4pLZh&bZQI`js1ci?LiRt ztJiZ%Ug=NNIic(YUv|DJ@ZJmuXF36$HN^LbwZUX7M*X>GbS}y|nTYm-?HX3cm`10C zhr60P771vK1^s5oXI#zbD1OtrImd3|S`RmY-6 z_vq?7pHO=%(+7%swv5Nhae3KF5@O9~bSnetR{R^?{)n$%bkpwFF!EH(Z8J)K6%C}6 z7!(Bt(fm#~G=Bj5C*6?1#m_%x>WbgF29gU9K*c_dl`<$I5+W)h!1Rk5NQfk-%y0#> z^mDx6kx|zI{{Jv2BPFK`rey!`82mM^+pk9g>x6%bLB^Apf{#oEP-I*@!zMZ;1N`_O zHin`c*LQXK2jv5ci2#J>mSwbY(us@LLy9+ERFYzhj5$(c4qB8|Otl!h=~v?(KdvTy zd(mfjhprt;@;}}a%qy*X!^P*(d;fnOt0E{w7?_0 zpLLt#tXX|3z9*jmaJD1w?MmSh9x^9R@lFaB%%CiOeYizZOwJIBJg57q)7y|H6g8}h zDV|$><3)3H%XU=^hr~iu? zZMN>K=p0te^xN`Uk1O!?NBrfp#;+JUywPHG`)F-2yfkA{x^#&N8c41Q1d?9`-7CQM zKgJ>?ygyO!oKcqH1K^!A#;~C$;dk+;cZC6gVxK#V3{ali!h+&}ar?_>BL;YWornyj zN2c|ecDZ*b#-P0A`0NONg_4`A;lmNWvqwoZlA+uC3iaXh6B8_FkOPqr$l7VmBZB9W zW=I<6ERG_&9KzzSuSyD8=^wN!E#Vr5CU_?}aXU{;8kJs~Fb$`DJfwhWcLzp&fV#{B z5m{*B-4_|jp5i!s5A*C}IL?wVv%YGtku2jjYHXb<7gjy?#*`shxH^}(nI`RhUvI*# zYZ|5(Ga`lBq?#Xoy7$$$urz|(cG@x0LSZ!rKLZIq6DPpnoPsAvcGOM>(SU?>?ruiH zsH}I(9ZkLxP4gvgQ+@EzrHf37@7tvz?%MNqOr+zByAx-7p{-AMTTCMdo>)35I(W4_ z6W(urm%(!kD^XUl=V@Cc@WPuwzX8ob_CT;glVNS7+cX8e?$Vqqd5AXy%vEvR#KSCY z{^FglR<@04A<_cnzFm4+DBr&B(&-lan_|B55!XZM?9Z_25;0&>=RJI-)KNoe*_-qr zOTn3WR5&5jkdhSbSdr4V&G7-3!Rwu&8(*Y-9G6K@ecyCH<9u(MllrqQvSII@m`S8U@_R9RqjL%UX?d}P&8=dt4< z&Yq?XF^^P>xk}AbcLn$KPt|sKyu>#(=`H9F%$i7^Ln@Ca%#>H=G6P9Pfk0BhpYsX= zL4*d9GK2f*f6lHG^Ouwm+=UCKN`EzfJyEfKc&`|Ep!;i$d=sz)ia*wekDg^{2{@bkgXn_aiq29WR(rVmB$Y{ii?bD1A9Pq)1S z$Z&Z9UVbP52tY-6h57jmGXe=AU=~^jx)e`-WT9BaR&|Coe(_=L2aCwle8}gg{7DvS zelC~O(Sni!#Gs-hV-n)XTHmxjv6MJz6&=jEp_c%WlLHV!-m?R=+9!<>mKL-M{Ir@{ zQv863Ag$IFDP93VT5W54OD#7ud#4jSRz~X700xiq3jWMU`2?U)UV*cWRQmUuz(4iQ zz?(onyB3rdp#FY&(z9Ks7XiCJ?44*&-ooGJj}11i>AUfC1qncnk^P*(5?74-O&FAd2&l3N!? zm$p81gI(%kUC+c>dtlGK%+&VoNcx%Jtz0(9VB?W*G~q`|YqjxL8Oi=7-^gl+9DOzK zn0=b|dQAgcLf=9mRo;#$qfF&uxao~C38EL7xqFi}cc}H2u8>A~ ziRgPOXs<`2vOz8>W^2;8jUzYWFZht|pjeZ;_r>MPe%iQ_9xX-U?#luBEIWLGBVRw( zuEKymucm>Hae{zX3CvCDx@jTdcbuy)D_Og`zxBjFY%7jeO%fzq8OUU?=ojGdrb5yP)rQx<||H{$jUX<+G%6U z)!7ubNeVJQ8@VbtFUU*Ob6`2sD z;=2?4n3a1-lKQ(J;g1G{yBw#=ga)(D9-QTm_rd%z>~HeFll2EGrW5P-yjhD5(4Oq4 zMZO4-{xoTk0Ic(IeXIvvlM$mx2V5Dy$y;%KXE zbCWf-hcvJZBf#+Fd>UjeKm$q&-^FU)oI@=+ zSP?Z1EXinEIG8|TOeSU*e>~+XI@KFiwZ6gk-T}(YCIy}Y_hS-on;*8)XX>&~va+>2 zUq`hvF4hTflv>XhvS)EkR^R; zE{SMH)k}_6(rw9otne}&x*6oK)y@F9mJ=#8;B0OM+So4q^V@tXO#2=9T@K7XUY;wn?AX_b?eKWtw-TleyKERxO_C~ zgjme33HJMXc}RDTWXglz;eU25eZI!wAAFA<{M^?&u~t&Qf{PsD((i9RSke&N7||V? zXXRQREUj>#r$M!hjAI)dTe1Ey79A#!;5{xY75?7GM$couXjWUZm#2Go7>Igv{Cg6Ubr?qN0HZT;U<~e@u0PhtKx9S03D#bFy=dHN0ua>8r&SEXst0gvlduQ)%6m)C_+||9%>ek5TmQ=v z`6HRmqibb!I&i?**h1w4_f3iU8h9}!uG;U+1Q$5};F6#N9%rlo8Dk4r|C@x7Iy(y_ zjKKebFeeeJzz@>R?O(2y{%dmn9K`P50Q|YW3NU>>CDt8-2Tie+j_`b0dPHBVk+*cx zpGwHM@l*->0Ik=JWQQH<##72{hfen{JY}=~YTevNg|;Yuz4I;B>@}lN`>-#;v`qZa zo4u6;368DmDmSRG(F5~H>hpE`h+QKQ>6u=~TT3W-9J;s>jeaXoyT|EO5Yj7Sx_Z6x zsalLx8fV(adxf({5p@>KVRjGeQ+@iKFn!kZ>fF_84EOT!%8VDkv1*5J4BvBxS-$Jp zaCyw|rHLg_wm6A}qJWW_xw&eReR(&n^mbCfW8KgKOu1JdS|c;&;fIGgsV>i2JDQtq zoX6S;{V`fA(|g0h7;DuYD^`10#e8;ow5W`vnJeTQt5tZ33SAL5u2;ls;v18B5(}r8 zgl+C;tCv!}YL${HbaayaRfLSa@2NaRXVQvh`JEq*GTY7wyB6dis?|$e@+?v*I&6*i zUd1%!6A6>@^b%ft^l5_Oh>CTv<6z-4-1Ly+t?#CMsyin&3n%lqBe6I2#ruLUVu$87 zNs|n^oHm3S;RqjS!PM{Y`c0BdtC|U&CQ-J~5_ks3YP?jH=rA!HT7Clq7qdQKjPSVe z$(t>`u{z{0fs?!+ki$0+dW~BS`WgHXux4OL%wrptFzFn&FboBbIY06zA-G@R3A;Ml z95nu4nl^DbW=9V5d1Q%|<_U4A#)X8*GdXrwM|{45o_{1rEcYM8!bu@}aL~^+zdo)$ zg)T{Ec%E1%tvHtdS7I>&^d~Tb5K{akmXxcjy9-!~DawBDe)&;z@dCnNrFr@THr?<+ zp-`|0`_Xjs-vH;|F{^Pd+V$3C?ToR~Wq=;A=38mWW#WECC$@R-twGBtwp=m&0$6fy zu{TBVg;s!Oi{gne$+B*HqCHhX+3n3P{DB$M5tPfCxSBNWX~fyS4e9xMBnCPz&u+y! zwUpJkC)v32#=XIl#pC`ExP=xx=XS~7d5R00VvTnA6)lxH8`tXxnam2zL%ZL_ zn!JAy<^QfS1}+(W`=w_`Woc`L4Vf?mFY@h>&=YQyJ9l>i#qnaC3UmUcct^*PUD+LC z5Pj%WhU$tbAnwMVR%4=2*Nc9qAt9HTfuuy=LpX zFgQ%iz*rg<7M^JBb!FpY76FA-vRl`rGW9%;dGXxP`oo8!X=3jYX5^T{E0bj}a`-mI zl@FNNFY}GHeNaY5=($4~|1xTiuB;5ds@89FIKqij>iW}Cb28a@a@Y;uUH_891&bWM zH{ocX>Z|5to{+uB@j(298;4CtRi@3x6mB<*qp6|b*1%j?Ddj%tRcIP1@v3f63ZKAe z7EgT{+ScVlLXYvmN5x&je1*=ml~%*nv9uuczFNzOoM|fU*G%fRrl_n^Urgqtof&H$ zI2G5KY~$F_5>`awU-+p_r0zY2f$t)uzgxEl>clb z7rTE#r@szk2QJc`7ND2_xCZ~DTj(5+&Kh-`G8$#B>C6k8W;}MI5UV5^-D}iCt4}V> z4C7o0G07*drQZ)Dx7Ma#eR9{szMhefyh#^?v5wrpqK_ohGF7YCrqr z7cSktgYCUSz$^KFnaNP!8yVRY8M@^Jm&_~WN!_Vxxu_6A4;I;>dA|jNyp2X4tenM7 z{>v+SkdCmt0Ts!M0`fN2+9fYCbswxe85Ub8eSbvxcH6`7ns$WadZxPAR~W{v;B?2X zaoR|M>y${4ZrYMU{R|vQ^ENnrbD6=8j%^=HfgS9%BLmmAud@wsYWXFKn62fP8u zfBJ`@t$pD)j}Y{=&)PLHP@uRc3%KWOJ+SNXm!XJ+4W$IYhD-qLLmM9qS-UYlAnrZ0 z>wHB$jBz;BNbWBt_@R+!h$RPfY8yk*1BrT4z~{pqB3#RNVu?xaUs(mq5CNg{qw_#d zd&B_HJ%_>4^ES?_<6AXaL*(CYWY7he)(L@ zRqb%J<-lM2`dIA>SyM0xGevw*aeHa+sw~iYZLE9QnlT?uHxKj>G;8N_ETu8XM4d;cO6nW z+b7i|G_^jFa;tA%oX?D%S6)0Lv-pNRLu`-1k1)4+uVJs@PWJ-_*Cx|ty{XpywQ8La zCdv3wW6zT7*}7?CxmTwJ=1ZD<4?C_Hw37J71>M$-2yN91n!7-U1XBgSN9b8CqS9zL z9mbw!+UozLz$K2nqqiGB`I70&nw_o*?aVysl@+u0sK=N?En28Hvl_+UN*>&JB<(Zb z(-t~Pt!0q?rDTjC$$|?GuW#dCR;nd+6BqVspTyy8xvyKa#6)$+r`$==<*u?|WX~2g zPas(Z%10TCec!1Eha2Kl+2WQLNh--6SD-Z)!QA;Q6h;SBxD(!M2s`H$`8W&ll3>+T z2tOmuB(6_*gN!|qeau~d;iaPIME5~9swdbJ8@b%Mlru~CP&4UUZ3 z{0#amdI(d@D)hrkpA7t%eU xMC==sU!rOR+{kGdeGcMpuIHU0~yo+z|*HTR83Y{vV%ki&g*t literal 0 HcmV?d00001 diff --git a/res/setup/setup_x64.exe b/res/setup/setup_x64.exe new file mode 100644 index 0000000000000000000000000000000000000000..ec2ee74b4a8059c8f977822dd155e44d80533357 GIT binary patch literal 152880 zcmd?Sdwi6|+4#Mi>?R9j*+p3m0=nX&(TE1(A#p)>VOQ>j4a5UV6@@m6RIO6j6|5W* zH>p`}S7~djwzksKQ~UH}TdM(Un-ENboWTQlMD5c~T&z*s5VW%I_nLciz{B%?-p~8{ z@0SnkJu~;rHP>7-bDidJ*IcvSR%o-?9Q@DaY_=Aj@)y#-|ND>EZnKR%Wz$I86T^0# z+G3CIIQ7aUUtR8By6pCwmVNna{<<&UcH8X<|5tAGFH7F$|LSf2$lNRZU%UN=8_#jO zhX?YYm-HgNTsrt0T~vmE+; z<*ag^XFpsydk)VV#$7aPl78-(Ezh$b{<(hMFs?$UU;fp)B~sTHwX8O3v)%B`B3t#h z?Kk!3>9I|)jVKs-rtK|KuB6fYtvrDruMUQEii8Vnwj!N0__Qr05#mOCg}h$W&|fHD zb^yp=d?1I0bl$h^wpt18rWQ)uTJ5$cJUq(^Y!#DuE-J7skV;qCZA*E4(O>H~>^2M0 z{v3AOL!kCWf9E7_T$$jd|4n|RZPIobBt!oE)PByg8@`(?NZ&xd5}tSS9P$^Ub>}R#GU_(A@?Pq`?=$&A=PX;ktWL@{@Ly7@Q&`DdgT1YAZs5h$H^b@(bHb8Dd9XgE|4Aq~^{ z#{G7iQMtuPN0-JrJ0M!UVSW%ZcjkOM-m}>%UukMhx-v8D-JOum$V380c;gsbfh}6u zYNR6pzhSWOM8GPS6FZEitgQb+LuBiGq{O7^rkU=}r> zvdN(_*s0FDNb(CDby+T#1I$#zj0X&nd-dfiY2uXuzk2X!j%x4Kc?9@905=-82$1&e z+qcKfS7YgGy|MK2(pY-7>uMu)Xi+RP!&Ui8EIng?*gQf+#mpXJyQ7s`$=d~qX^{T#b#sRA>K#KT~haYf|P<{Wi;DJgQifTHs>2f86DIl$oab8 zwb{(}cxFLh5iKvJ<+T3oh3(t*SeYkdp3%ekW4Nj1Z|PbjGKs#(b1Y7iX6SsZd;s=R;+WLVCC_9&aCk^N2k!b zTGg`)SXZ|i8J(}P)iWv*m@i{sNzQjKNT>R83CCmJ#JKRr3aX7~oPP(Nk?|eoQMone zdrH<@^T$l z?~=%zE+OYSg6fx2#bXlssr=l*50(8mO5hhG>+$O)7IpyPUjYqr1=bt9SlR-avu-%C-9_g!&^|RY~A~< z47XYt=1GQHGITn>NogT())jinuIpO2O-+J~>Q%^QnEQ=Xl`V0y0Ug$A#1h9bkHeYT z0!E8jnDf0!pR1N3c}Gj$E3CXO!yKW4VRHPFc6*-MuWp_uJYiehY=ui^+#h%6a%#eV zaydHEgK&hzwV}pb(y6*bP;c+=?P?34%=YXUa!7xXe z6D|Uq^;UnJOVHe=`pCGR=g;KX-#=LZ>yG#aQ>1^a8Rh|*tJ!X8{I;C$Wdb?h-*mKV z(=dlkI@Euyk(;<*(wKEFI=5=$i8jf%oivDEtVisQGil&84=`7E&_9NWv@p`n@ONc` zN;uVu=yUEN1H$esNlrU|@EwUU83{kin@(qv6sfk;c{?HL=uYRI`B00f6UHsA(&=B9 zlAR8_bmm~X>3+Cb0LLBv{zSWBy0`QvQgDBk?#V%d^|Y)CKFiqMD1a5Tvq@rEd%l@v z`DXr)(#>>%XocCO+qsPDRZ`F)u?+0LB<-S|W%+h?3x1|^x+JHae?+nDbUw_l)wb^t zvfB2;e5fB?Y1?;6Gp`I(80pb=qNfYSHF=E7gsEtGE0QZj|K)sc@H5P(`ve_CV69>9 zV$kM%pZtRa`e%y8=R{P@Hvn84XG+q>in!S+iHjss#sC+#m-FqEVhpk+&?1B+lCg5t zU~P};*MF4SDs$8$RW3ryfFI`Ts~)UxiPX1fu)Yp0ua2$ndWz*bQ@Mgwzt=Thf30q# z@RaxH7#eWujfUx%C}PGi4OiA_1iKQWD_eUE&+px>--Dt4bBe`y9=B%?=f^wTqPWBP zOWxYmt1`hmVUCThNyn7d>}d7dh>i@a+^Sw8J^Rk%cHQ&+^P+y*QcBv)1Ie4Ade2GlV7_=Ko~2y+SFpkwdSj(akH(Z<{F!AakcZA|3P4^H0s3{q_C4kR3ndy^$<7D51yTQ|&R>u<0gUyNnHk(yRO zW|X%B$znAwOt*Lpb05?_#4J+xkRWv#p2$7}-A~t&mpxCTtr~~-k8M4?Ki6e-qzvZL zd4_oi&=t(Y`bIE&ddg4UnDlTt-4bk1CXCdLUS>>>v1qv)0RJK+-pQlpK1Fe$(g-5M&&&o43@m*LHa6}%XXZ|`6`WPS$hgunc z=7Buxd*^LE0UhSH9cOMtDBIK)Y7kjg(-Usl?T~r(EDa`_^EHv#Fnju)TikCS+vjNHQBa-+{NRnnF2W z2SKK}JN2S!HA+#f6Q0o9TnTL1DJ<`}~~(h#g${tPdNS4SGJ(+o}CNH$cp-36Okr_Y2D(m=E*&F_svP|86yIRkA zZ)I!cR?kl&W?RJE8*cdEtGC@43pe!Kmbmd&6talfQ@K@7InPg`BX%?E!wv7xyz#bV z`nrEbN9;-0^qEJ}^Zt{()=0U3Qx8*`*mFtFV{|&dOT<=hvrTI)vfC16(-svK*b=8s zTPlx}oR1&*PcGHxUlA}+ZY>dScz)Wpe00Qg-m~6r3#Yp5k+gFa!E7;U%L!nIgw6M( z=ANi|Ae=f{yz0hqCR{AIgO&LK2!SsMfsSX>+ryTR07wBqE~IR#&%SC*3;kis7#~Sv z_fm!uR`!+vFCxL7RmWSDcxC8ISmHBlu^C0{F;WMQb~^t+RAxq}^Vft5Bt7kXn4o9V zKSnPfOQ!caoteQ`YY3*tH(D9SdQ#;SrXu@pqwOufF#;Ah&ri{Do>Z^EGNb+p?H9Fs z^u#rtamxMSV5-a2>8vJLNEX^eDBRoTPh6Mk@^?D@M44Q?IGNB!o2_s+XcrNVoQCms z@DmX1TSF|kH919Bu=%3R$y3Sm7O5%sKllxi@0IlU0|d$a9FJ@<8QKVVHvRp?MB-QI~go5PlRvpDVCBz}m7tz~%NjKh13)7(i7 z+kG1kI^>QKM(WMK8fo|Sd4Rs<*a%TS^&C8Y`7j>m&)^M6EC^IFsmz2{=e^q7v3~UBrOk-tH!)AW-hF% zDgRgRrH;j6bFJZNdSCZtLNv--16AESiQ=ay+-kmOMz%TP!5+`uEs_@5=8{P2S42{) zws~!y#tl}m#GV+->ynuHLacC$u(*ocZd@46_w#;Gap6TxT%ox%(^xQ}uv@C|2@!rfyfA!y_`2`{@ePU0-$%>y`s{J)8Y@z) zoD75Pp}aOL`5Lz-hpQB?)jRwQsJYvX;B$#FsC>hWOn8`4zI&bPq7rX%IE|ph2WTW? zTVYnz)wDHnQ}dM!!PFt6p<4Iz6{$NB7ByzrKgsX>&KP9HijZN3D}=>gXw=m@n)W6} z8HJZs8D^|ZodvP8{_M%UHZL`rVO(M^fr?ncI18bD-Ahj*1q`kC=ibWuLC^!$e=%$f zvvz)Vm{EOF0x@zqMfWB@$V?Y~9d#MsTkm$GuD~d4OMSy(TXjObZkEHSyWABsONPbk z9K&l0+hT?2S6a`pbYEHuV8nR1<&ls zUT34V~?E12}He z{q{=nkg6)OprWc$0kA9Qi-U$?c34^od>N;Os3y8?Ftj58QD-9<)6Sm&Ks=xmp<@T$ z1$;nRUQqYQPUo4BN9x0BDpoF#%v+}#j_F=3Wsl?%C5mMwq|`j(YA7q{dTN5emA+Y@o9rz$=AREQVNO-< zjhVZ`EwX5o^Nj&Jt2dKta=z1E5{L8E$w_#zi^&CXx)j8?mX3qwCqN}#8vh(Q^W`p$ znH}JJ&x_!DPou8eNImO_1rK?eZl!x_?s-DvEizb)GV@k(k0emc>GeTw%lH9GcqkOA z9H!l6b<_q+*WrayGjATlAsD;oUJW|?^Fdw~&$!P9=gO_Pt;Z6e4dbK@rdSh{!`9s< zHH)+DE@^!}{KkuV5?SNIJOCK+>uiJviPOECFp%Eip~64{2Cz!{3r`sW=(ifs5ID~Y z9EOv?`Mm|_4+7^1*2t&eyd#avBXpI9GnAjc3{vYUX&wQUdju*IOQ0?naQ_WB?09+& z%v?F&kARiM5;y@k)>H$q_A`BoPlU}L0M8Y`)Zo0B00s+HOqD7SbT&0dP{kdfVULKI z9Xa16dGL?E&<}pdOokVkTjFWMA5Vu+`f5?E43~J_mm(5Ui@M{=#wxyP@N^IyMdK&Qj=A9+x2Zq_*luMLn7Us;4)yp3V#L+(h z+#;sndZP}1Lt$HRsbh^ZxYV_#hF5A@YWl=%U99d> zZ)Q%;d>quRdaWS*|-uw$(+5}drs8wq~hv37VgxFuq~6Y)eg z=kScRjSjxB48vt}Jk8R9RwgBcHhFCYIYp@qd!l-IGK{Y6@ka1ic%qD^jtJUBmB)*I zkDIcH|8u;b(La@=!jkJDy*~LSPNRc*SUJaj9T}|Gusv~e-U_77#<%fEc%jGvZ8Uhi zhSxKS8bf~1sF;^%g=bXQPdr4tf_N?Q5b;ICYdzlPdLWauh@_>4Sugqm(WRbIX?bI$ z8zhB_s*V#@JiRl8vh#%OyXw9HU?`>n4!_K8=SZ(;J?wT}9yT*X*SywBTar&3z*|kX zVCm^_ZQi#ly7yUDx)cCg_$JshY|Cl6?W@ehonte5!bGmCLclk>+=(Gy{ol5U^Qw3-@IX zUmmaf5+41_U9&TelSCYbBW8Fs!%s}lirJZxZL{h|kBSxUHG(fMbA@wl(O}myCH(IA za3+?6c|3>pS(?X|rG@Oj)rzzAWE|xA22;Isz)?_9BOD9f|3TdBuFQq2S2_}&n!1$^ zwHMhL%Zx707QyBtCNgBNr*Q>(MHHDeO8rrb?JlU~!10(ZM97A!?(}*Z#eG_Rmn-S2 z4tg+Yi3C>-C!5PC1X53LVSQpsxVOlgINLCt53hzJ&^YY%|Jp@=Ag3D7rHs+k>uD70 z%1BeFE4#3_(3Kb)Gi%DAlcy1N(8}V{Sw?0r0ir!ID`w6k19<*hI#*z@9&4+qyA&78 zPW7KZu!SJCX0*%GBrXi2`eIMxPb6TDjAgEJspsF9(OJ}eA8Fz0HKhqptnT_!^%@JZ z@a94mmdst|8z7K8FIK&}%u-!-EhZ(%J6ga9k}55r%-vc>?o=r%&IYAS;;ZCO+`u#) zrJ4jTlCbVlR$(j(${?5-URM-w^zI+K_3(jQ8-7Oo#bzgbqE*IEt&Db<#40VHDl9#; zU408HTHPx@_sGxP^0S(s^=gIu4wd;<|6e6a2$M5zzF+xDxVm&Tjr$;59IIPOKTT0P z=-rgEC#K@DO5S8-uBKUY%G4iva=EzKQ@yYh^W<3>=yeN0et%f&n>At^p>6F{Z^3P| z6$8cY8d~hW!D532eole)*8I@w80`zmoC{yyuKskUekc6JC3@LBtx@!Qx)#OFUk6&D zKklx$`Bpq*c;PfP=Cd*LrI@*0Ek7(M-y7IX7Em|LCjwoP8%mFa+t5j{8}*uK+2leW zw!yW`YPC<)vAQ`WHRjH^*%)}77ieSa?hW)w$b6xhWd`7j;55W`4WaN{Ix-PoO>$*dRqxTNVvi4cF_@08s;-1#H4ZN zZyqC5WA?>?&*CXk<;uWH)@d~+U)zB0(ZSLIlJ0#q9VUq%8#B~%@3rV4!I=(E=5hj- zOW)J96;R^V_cZ;LkU3+(TjRM$TWP7gDr}ynpYl#!Oa~gNm3};azeIOe=SbX~?^hFb zV0M+%YLA;Mk(a9iOVo=;a=HAh1S5Xtq^MRWA_+NR`@2 zfd11a&!Q@u8cADlX8Ew~)CW53X}p2b-9dih3>Ii{DJpif%Sf+o?Qh_pFkY+%{)*5L z3_SOIUtX;(v3b7P3cbyy4Ybn8%(J#+Woiz_&7Cq&{86(**%5ReZ_Pp0Y}*!lmhX2h z_N-||^{Lq%jP`j_xSOH#>ivsjo;BzRHO=S~j42zAE^ioBl9kmMDp@?^SQqwq9q@^` z&6}=i5h$Kf(Z|A=#rIDF$YO&OqfPno$$Rb}E90PWjw@REH>*7*18xJbFgqH&>8Mwor3c<@M1L#ZuFFb{!p$-wm#R_yFG!4-XM3}g)lnvhY&lyR zH)m8Bb;B6ttgSHycT-EfdXN#r@|HiG+hL>t;fOZ9yY{(oZc8M1g--&gw zuIV9=fo^g0hQLaCE{+YfBxXL35TCz};lZ;O(rge!Sb5t(ZbBivxsORw74PaL0NaSd z_O-;P2UG!UGK+_ zrEPU{{j@20syQ>zrV1WoyAXaH+NW)oEg$4^>@RBSAZZ{&(@2P~L5xz{v|hB^sGA9t z-LT=kVpOdzkpbyxx}Re4OijIPm5`m0+g0I*&>~~eqD#{No+!)$!%6srA#0o=8(h7L z|LEM9W28#yZ&yD#KvKgN;Vpwa=-TVr|A2>eo2q(;x0#y9)J;V)3w(jA z;XM*hjYM})@17>VL4lfP%1@C1<7El9aGBbzUmh3Gx+N?S%Q+H21;isfluxs63~CM4 zOjl+D?3o#c`TM@7@pxvUdOe%VMd0HVAQ&}C?@dG-F4peZX$==^ckTHGth$OqadSrmia;pDhNjeu5zUwL^IDCDaWR9MSStr) z6y6~Wl_HBQw(y`}qD#pB6~N`$0rUA@y_HkFgNXNdt;nZY1G6b;$qY|wia1>; zL)e9|3LxWFSZ5TzoVv?lOH7W})j8C^8Ktb$B*sRY{>D7{J5i%r%pJh>ol z{w&}H)!xETa_$hq8s-B5(Yv56Y=1uW?&MjE{mjHRG8UdfyO5;dB)x&nlcIl?S%Y*k zJg6UbIM*Sb~w!N15&HCx{pB^fZdY9bL?0)U9x+|KSCc zDiR%UoB3hYmr!Ikk)gM^J~6yEYV%quBl$>F#G`A({u^`_At~WMTCEaW z<_qYz2)dy`;H9z1sQ$qQ5_OT1W?P<)ji_Yxn@;B^wK8N&R^?YmDylt6Z)$C+Cppq+ zSi4AUYn4)r#yt^2UXro@(<-jB72m1Uz-cWIIGv}inav$aU^ z_vesydG3COlIDzlA@e(87G09JL6gE|OB&YtCF30n4RdBiYNgATn5XKs0yUStc8HkE zT=BxaiIC^Xdjq29!R=vZA_P#BMPQ(Iijum46tPU{*hV#3qg6;P0>a3Gx=5@DY z9WZ_dd<3%JD&x-VMOd*VJUzn5T;Vb@hRZXmK;2UevKi+FUK(lUO5VF+q`oYhtDFz% z^f1|-t91~G)Hbj_AsViRJ^6ECmkaEEh>Yiqb66fms?F8?L*5Qu*ULDw+D&uBNZav~ zl0|)kEWPb)VcAqFG)22yzso`|!+=-$*wGT6HxmYK`<0ffgN}X=nk(cA1b23(Z91<_gLt zL|@vbZpNRao)M=KeiikE4zQm1D+!pje%3tM3&k!?^{scM)`Obq#;e!PPmJrdWDq0W zrtZ@DjipBVUX3IxDWu0_;Ku`o+JLK97Di+|8AxD3G`kXqwuoSxnnvl=q7YduyqTn8 z5wW*LREVAG-){;w7UXy)=ErYQYF_3rhh^RI%p9VZmYP>N%#k_Y?Fg?3dj9DoZ9tSX;qh*A13b>l)9Zeel@F?HYD zLavp5)kGkkUP6;(*y9U|nMJZM7zqQuG8UH(ik883vA8)5r${*Ew{4VV-*EM?p&MKu1d9Xt9eq9*?H9z6YV^N5-Rk>aM(-KMSC zePSXNG7?xRZKk4Js3L`)(wG&*g8%W{_Z9(3xoz#w|(fF5HY??z$s1e=Rhbd2~-E|K2>zsbOQr>GbB7h+}~;t`jL zs+1mJ{u!1DfEYnRJ<3MQfw+1E%!B*aoEBlgNlr`;^#o~b+#1XzI&IcF7{7_Q_~p>{ zDQ3xWYb%iENqK)B>nmbZ_=oZ*@OuYmp7w*uPM7Eu*2=4=cJeb3J4x`bJ4y!tRXg5jT%g{Mwm?! z^1~GAgr&0?y+Mt7P4kaabI%&7j?uw$SN(xy@=qyFS`x5~{tv`a3{jms2u?AxHEv=K z?v9#mG4`bxsXJXF(;nEy`1C}>E@a1N|Gs#?hL!h=--|!nlKYrfRS~)*WUv%rwyEP@ zhnBllkOv+`*;J3Na9p^hKzjV&w2wi~xf=w%OUMTe#Rsf)<$7?oY_UXKcHA(3BSvah zjx&JuQ>$#r!$xMVU7dM?OgX)KVq6>+nz_oo4HpL88SCt-j9>b4XCD7dhYm zt1x0&02$v~;7ycL*hqB)1H9%lKtrLN@5+rfIdTM@{IW1IK^71EV3_lL@D$ZZI&S5p zlQt)vf0ezg);2VV!{M>GXhv*`(-J^PS0yDvY~AIY&o2c4?OH1C*p*;fi=tq)2O^-i zmk4UbH9#KQJVyOxtj+c`B$QKGcbtTlnAd{h=GJ(5+l+M%SGd_8%e8Tw=Wj;p9eeT+ z#ghjp*(fC^P%^tEKa>W?2R&~bK4AP3CB~NUEMC9PE8Da7CKf(cocu7>k~^yZ*2JE+ z+4Wz8VR~yA9&Ce-7d~ek-W#jj6FY5RLZL_WQSP$F-o#9!?Hw=SYQpE{!>1BH(MlfP z*qhwQj%Zg7Ru*v`)xV$U-~Z^}kM-|I`uB+b?aQwjRlY)NKTDdQLXLjYTno?bCr#Cu z#7IL{ZlT79;pQ>f7ob!aN;TU_hEOl+SMTylEfg4V7TRTc{4H*VT=Z<>l~}6JnHn@uO^A@Jsa#w1lgXM-T)Pcf@YwKI$%FW`(j=4zivdu2}VMA4h z)7Iu)-o!T=sxXFH6Sp)}!K_;o3mU4(-msQQjP{S8%56!09? zP=&Lz)qE)#+5{|H_JU1fRAtZI)Fh9xP3z=w>ZV3{oV2My9>;I`raVeErR3q^pXAkD-t0-zR5C1osW$e_whlG{=-2=Hlgy@DT-^n5 zAx5d`oWjoR_U&o+S^9}P`_wCNDVe?r#2J8K-N;$#>*(JKU&^ ztcP!qtdg&Cx0N+n$hJdj|N52l|6bg@U(4Uk?1o8FJ?D0v&*-x3mfpQFL|amCT~hvB zY~R+J%C30Y{TSec9e;X!4AJZTqIxc^5#hdHbHpRDR3KoBb=u)_tR^v^jr2*eREH}x z<=y&ACdAEL(i3WYH}vF$kmrGCVVK#AVmZVjyry_vadV-OYxn%+Lef!?%c}HNq|u+w z5X-DB2r<9L0Vw(3q57l#L=nHkl0OnK!#ZZZK^-vm^w_{ z1fA=Ov+ql?xbSOmPZY<}Cku+PT-2ty8U6z&`Esr6JoQHwB>q)@WI}RsaavLG?^qho%zOk zROmw8&f|4EbDcwJkx%NrQRKb_o^uBpe7B!8K_8#U={CV#m@AzA^Y>g0$PS~0a`Jad zoQ=7`uI1Bk&ocjFhkN40U|??+)_8u`r8(v9Sg`Gmak0X!nl5U>c%*}{1GSzIQf!C@ zo*yUT!IYgV)+r%OKJ$H9ePk@#GGROr_B&2cZzHiX3r~Qt|8O+i>*Mb8d8^V>0GSzo z5&A{AMehMM_i>o)D*PsfdH&mV%!c&%Kg)wgJ400$Cozpb#Pe;ifDyu@9Ma-Rw)dr1 z8{%``slr&@adUMjoOkWaGep<_P&>;u4x(uLM`L)f!qvr=42dwvN9*& z@gVm@SMT9k=S+c}gB{TjeV!g)4N7TeIlq-#vtMOJ1fjPY2DnYFID^*p!1dUpTT`VL z0Gniat7-|GN3)k0CfUthVzF0WB}DqiJl7wi|MG zOJr&n8dGbyekw=pdSkg({CeSa2-4(BkYJMwRMhX0Qh8iQQ2$BPOGB2$%@5ZVF|!sL zsU5(0wtnh4qIKn-ojNUXzNL_uJ5)j!?i2YlQ5$-Ag1Xc?);B7*()BZA>5FCBt1XhR z@ZZ^AOVeiPk)R$YEt{drUuYoOnh43EA-XF%TF`yzBD&D2e!;k;(*C{9 z*1HMTJM)-#Rff1Bw8frr_bg=O{EqkQ@i=8A@)sR3-T#!xWm3{~zf2h5Ro!9Y+q<8# z{8fWZP~GQ9UpTL!KBno@l3k7tM)z~Z-1B7NRR4-$bV&t=uM;OvoscXh0B`K1xeJ-u zbb>H<{$=+{Rs!n^S)cq+Tnd8&N%qhmLpQdEK{tKWN#v~Y@>Sov`2D7w?~lI)8|%o9ed5#j&7) z#mobku;y^}zdXD{JJ+fpq=b7Kgz;Rzs?raIiXp?SY zONi-nqw8rThwYQcW2Ti0eKKwb5A_S-cNt%ql{7|@9?&dshD)-K>~Q|)Iu0rs<_vEK z4~8`+{aM4rm=}wwRIf}~9KDteT(Z`Ht4KJ{0e0CWX9;Fk{EGVM?$#eha9jA z3?8r>(|^Fun?GQeC}slu9wK!_%fLE{SPArqr=zRUhwdacW3Aj~01 z=5u)`rp_6(MaB{HPtC4`J@(nzyWu8;+VH?JX94ql-9OtHt8fE@_05Q^>-cK$+zEAF z({P^z>cVT>pc6>_3s}_X{b-!zfIoIOnS++j^^47?bYU0@Sii zVy0GZ#j?tE&!_Q~dv(O#J(~!)Gh*b5h3=_Dr5)&cdkh(Ad*ap3Vjv_>4VzspA)ufy zmbb0zLSc-dHl}^u*OE`Tj5-4=$%0k6xBCOl5Cm4H!Pc6~{W5n6XQz*AIAUwXlp21I zyNO{4bM`66NVfIb#B@{7AXr-@qyhHNO>IvPOe~gUk1(q)MmE8q#$S${GcxBF&fy_m zcANt1j`mSIofwKBFX9y>5*IJSlVwf-@@YG}&!Zx!^uJa%j9 z<6PnbRI+M0#OhCdT#)do;qHN!w~w&I>i95+>avcoK4GXqWQ$Yo^X77LAQP5D#d_N! z`v2Y6u#q2Oa;S`XeTPr|1=8iIxFQeBb&q=ixzP1Bf>7o%qLz!gEG{(6g?nKary zE~NJj^JBv^vrm{pA)!d05&U@B$&pMHDXO<6?3pezg8fSB!{fAKpDZ&ncj0~(O(LJ? zHv@TgS$RB-&+!)Q{us40C`w+BWyT$inY~ElSUCqG+lo1fj~iU7V={E%vgs>S?t%mS%^hZ!X!*=koI{hV0+B)2L)g zA0kesFgYekj(!ll-(^%H!SFqO18_c1|Jz7;E5*YyX}1}bF9Si8-Y_dils8OkQsT^|@Schh4SWqOMMxe0~ zk^oC({UOdUe=&X7g`RO2u#=5vSbk}`0$PS z@RCw^iGq*#;UQl23v2{o*m4&NV%`Bs(Q(3FquQRt_voA{NP?aqmJWlG=o;+)5Jx2! z2WZnNgxDh?s*rVzV=WRnyAidjM(`a(@cGlxvUK#I`bhQ%rnZDQGoNv86+WMK4o9TR z4F`EWGUGn;D%6Bqg}r6m=Sz4I;r>HX!go|V>zVlki}xAk_#aFcf^JhOz1X!)6^CUq zZBvKy&*icgjd&f~*9J~Japrbw_nr85cpad_no-ZoSIC*WlClfYO6Hnz~Q!Xp*PXa16(Wle?h5RAkFRiqP zdDzrJNlQ(9Kq3Qe&iFnNpJc|}`y^K1We5~4^BcD&PT<&W3j`FcyZh3CB1rCSR*5Z? z5H8&PJ1Ldd6@EHgD&`bSUZxKW^rD|(O>j3Cql)zrlm!MW+>~ecOc0}d_g!cg{cvsx z0S6evH)VL z$Rm^z^xFMK8m(j*WpGKzY^sv8Tg8wAtyx{fsG_3$pVpzZyw5A-N0jV*!x>&GCZHT= z(FYLX=>kHsN)=Df^v7YJDm+tWG<-C3wO?%c9}+jeDMI+M09Wc3NAksXeia*=>w%#n{zz zQM}uwBzc~)F9wg}7Y7U2_>pQu6WClpd3*AIc;$I&czK(a>#%_Sr5nSXT**{BfcqmE zWZ%N0I2h@#P}N;>$T)qMt7C@OECv8Oy+0T#`@1U z$*Kf`=l!Z{yvfp>ulGlQ=`zv_T{1FqzB>RTH6aKx4o-j`OMU5c3vcUA#$N}#!k-!r z^=KaAG%jYc%8{0)gQ#L*w2KcoDaP?lr`ov5x(bEOVAlO3>gjr9lrZC*Zyt!o%|FRjD_OY+aql9UaLyMcU1rn;qMldrE`M)RsQ;!E*;Y;B zB4NR08mQQPqPia^Ma?Fc;GS->AX_ok1n3;_XnKsXG>|7+&tga-=bKEy%B^N!&i4kp z5d}Jzplm0|5b&}{Um_Z$7(QDz3JbOFZ>ij_&iuWsjy@6)YH_@%jKe~x$kx@3qW^!Z zwqNJVi;AV(y3=Y)O$+h@3u-t275{tq-)Bp5BFH%s2}`1#9k@{prf93s4!#<#&@MAI z3;gR>s{%}6GuoQkAd~Eq`qe!wk(k#Vf*h@?oi1c6WBuwro%k=E_*tq&-tP2zf8{Vuo&{}ffRLuMc|XMC!|Yeas;V``Xn605IXDX^NdsethZP!5>;RN zK`-&{VC^^Ods=sgR`Z8wIOzJag0}=-z6{k`P11fpSd8y-4_&p?XM+N6HZ^r~?J9-2sU~qV%Jl7g4+@0#1 zwCsaS_}x_Bs1-#mFmYRZLt6w;+N??)85#Cm+S(pABW(_m^#eA`{W4w|XnV*3_iK_P z@fs~@ta0`^w#&aJ=<*(M0S{CEg(ws!&6H1-rtiL>{wJ*RT7x$@J1OhM!_@V8IPdUw zC>CD~=T-}j)sD2ZLkx(MhiE~^(K#|UTh@%hC&L&ODx9S62Kk| zbo6gp8gIspj??j<$QIo0iVHKAsy7EC^VRc%kwvO=FtS8#8jKj~H-nK{wP7%_QhjG2 zlCGB~rv~G#&_FFG4hj5u5b_Cc4w>BMSbf?9DGy2yC!SCjOLeR{nR z2XdXQUYaD$xnCy)-IFfEA$%eo`s!3`^o&T|$^K)fdc2%$=|`<{ugC_0ZOSWYT1F3< zH-kD^&UfO2h-Q^>H;#rU)cP6 zG<(gaF50aRG^wM^6Vv^l3uG`!@uU~0_%Jf#OqV-VvOCdER9}TIeVpmuOHOk9g9Plj zEQ-f2H5{^#y`1dA^NYdH-~`?Ac};_rhXGYjJ;(O{0lw#M1e!Epth!GF+%K#R6OM}| zR%X9ux^ED?1pO5nfkfR9%p)4}=^Aq>r!k*PK2X`fyX*xl2&s#8!6cZMT9{AL6>SsD zx2amVJlVfac5!U$@vP3!DE7<6ejO|4d-S_Psy*iqrTa_gO3BaC{a57Di2sEQpGWs^ zG8s*GqhuST`?tspI&bmLqIQ8unIPSMD=so&~(mEgpzJBJUHJz{z(Ea&?% zw|>ZI-jj8!A9Eg{Q{O?4KSK|)JYT&)^HsJ3i12g&1pqnUQVP7!T2Lp2j<9tRZX0GyI=*L$S|mK7Ao8iqea9I2kn6{epi{h5y$;6Ht9$fFzzRh7o@XrXj2dTM zg1QoEUOh`)axtk~xWI^=Dyo9EH&h#7^ezIxtUhRPnmV{zc^Y5SlpCRnkT})?=5;Fh z&+ZPAQ8CE3Ta6I;DdV4geynhsh1t800rCAj2(j&kfJo`&)dyp8--ek~Y$}f95l_#n zjh7$hrjJ$rzpjxY~F>M@DyoI!)XymJ$ zcw;*E%l-kIc*q{A3faWK*`Z4Af1-u#Lw=K;456y2XC@D84WCPwjKX*1hTEXJ!=IU+ zU*a@xWr(5(Vg3)>f1Ky$=UwV-c!_)Kvbgd~r z3TZ2KC-0~+x2QAoFXgO7+`PjZ$*gq6tM71mnxq@J0U%i(HebpZ4s|^6E7_k?q#ATh zMRaXqx{CIjjFTY@5$I)6DIouDp;$*H5n4LKred+~{WvUa;1g@}Y2zH@{ zwC%4Vc-2`%vJ<7}&J{jh-#2MRNh}ZTar?5u@VXA8{3T&uA!d?1jo$t8s1^6oQVGqk8T+n0 z7sXF|C(@h9g;xO9`^R&%TMuCMevEG(>yW)awXQ@#oYEw`pp%1TcvCW1c8Zx2=_XbH1W`1t)An zbgQ$5lHy&MmbpJt1^#jg3sBU>B9z;^Q*gT&quLcKKQCY~UZj1=$FY{y^K9Wi3^m!s z|J{-1&Z^4_5{ia1^+bd1 zg`GuA1saK}=>CLDGG9t&IgC-I*L3vy|2o-q+ba`PZq=7dmtr`#TwEz>w0*={=m(g7 zn_>&>GXIDvfk9R zpW_s=CEa)-o4KU~F<6wHqwa()vWlQRZOv!}Xnm~fYj1;*p3A)IQX{n$!Oc}yd(~}L z*lZBbM8Ef6M>=MfvdpLb&qJ-k?cFo!t3ezG_%6DCsPbKI2OiX3$A+qIE=xE<%{QZ8 z-t6T++fQ!xcejJq;_5S#=NZ*EmnJ6;=2E9|R%qh{UZ-7VTd4UK%H2}hjj=@gr+Y)q zW!b;TGON~$Bp`Yb50nrF4FSIzGm^Y@yM^cFd=K5D)uJ$`VpJDDsHSnloQ-O_=Z=z4 z)h%U-j8u}nr(eI(z=5e@SMC*;wE!0VrqvB#(KZvmI7ZVPs@tH}$kqrglV~o-8uVr5 zi^S|{I3$)X{P3sPT^2uFPygYhhhX6rr_8p@y!xE)V~_^@(cJ6(O^yED*f88a`7@yc zQg8|Rz)LO$!>u9PF~JWH7OqxFn9uR&r8c!ucrg5_BCO|cMA(*(@ffm+Ul<>CehO!+ z)y;m3X;no6A&aT?zsyhI6RdKY-}S$O4++=u3(GQ?!}TgljQ~b+W~5;tRWtUvtC0BNZLe`aCa&?zn*%ewO_jU1M0liBftpiK}ww3wf_!77j7Bl?6i7af`W8)dmh3uJ+KEQ4LA zW}Gbns9DWT@#2O(Q8a3DEGGJ6kBOnv;HB%WdD z*XbOSe{0S!VH9#P3!{+kPyJXJ)+gE#6dJtOFz@3NBvNgq`L5f0Bkm5cx~s(V)Mohl z=F?;hU7_y`On=AE*v!l-TKwU!aQgj~SLR3Y@>Vn>k8EfUmxh}ot_q_eC!=7MBF`g2 zwDSGr-~!^!>l&o9n;S<=k=e~9^pfpui*cr8*;eM{5MEF($`#*HW04eI@ZbbsY29dP zkt7^3V~&O+qGos+<+duBT`azZFy_R}#Tg=Jh(u%I7!B3hbFEki!S-eU=28!Irk5Ua16GaWLAm%>yk*oL)D zMy%G0jZBk#olsOgVSNverVnp`b;voSrIeDhgXV)mIktAQ&kJ|*Qkq$z$2<4Oh(?v%h8LzEBK)Mjp;40Cx+L%yy10m+ds0S zdJ769>L+Ud2jBeIiln2Aw2>XqY?iSncVxLPn{#7hG|AZ4_bBIM{HeY%D_r%%xW~|+ z#lc|3FaZwAheMhh1PB%vwB5ICI~+}S4;=094_F*c`tMkdHsDuSig(7kv(s9neN682 zrOTxSk##J{vJe$R3K1%9Fxp`OpiCtBB6VHQ2 zelvoSIo98MdM^=rw2l4-ZJ^B>Y7t~|!Iu$qdK6N4Fz$i6)unSI7@9UG~+wx5B z3K8<1`m=*kj)pkZRWAv3E&5nmkl86aWePiGqs-8BX7(>nVK(ZkqDKJ7ItksMDgBV< zg;o)>S-4M=y9-gCypq|X>Ca54kQu^7o=uNQhse~>s=Z!Yk=cp!3+6?7fd6Hx4^;|0 z#Y(qIz^l{l%@gs#ZdLclnOiOeek33xhr#A&y_z-af%WSCy#RTp_kpkv=lP)2q!l=v z`l*eg0or5}$-{zv_6F;VJ!dm|k+F|j>|G{q@Z`kBatvmIB`69-P@DiszIQStf#H%v zuF(EapV!4;yvHg^ixCH!+gXrL7rq7@dI}aH#}u(RLdXc&z;frSdiU0$vifPU$q*l^W9Si6l_p=~5r}==vqmSt^NXx1EPdaP0@o+ew%&l>Naj3QkUaliAbZ{I~+i zl6NJG4_Ynf$TRC&`$EiVDLGFbd_&37~p zV+${h%$(8zh2%fiGYv4r3({-);cTZ9n!7VOL~IX{7Lle4aZFig2hp zDT$uf=86}-7BByI^0yEX3>)6ljXg8;V^szmpu=c!^2qZBMo7Z{n#3xuQ`65px1G03y%TszfFLgMp^ zGwhHq&G*R|XA1_io&UEOGTY^{A-VK-(B{h9g(2()*$cdfyQei@QCqYM$E5T;xi8eX zs$fjA6b}{X!fen3Jx@6;SK?uhyG1y1zjeV&5>w za=t&@B|Yr?Dc8m2e7~_GKO!=?rqlB4*BSI(E9NF5EtQO5f~O>^&9;;}D=}qK0V#=c zc4j3`m!DHpeTCVR%(hAAOZxGX>W<@KOghfR!zuhoZad3Oj8hSregovhKc!r^{QOOR z-jJV{<>v+Y**)p}Q^>Sc%66pQa%J0O{=2TuG?a2SlrPxuf?c;7$*|{pn=5nR7f!4k zMD=SHs*5dD7Yfj8s5i zDeJ3;Id*X;WGEAzH zWlO#C_gh4E_EilQAR4T|v!IF230KqSPRNh%WS3ek1%^{aLJNC5<@jP206zcygW4}x zUKl(BpYgfzLgFvPwKL@V1kaBW2F;Nb=gBgN`98r&{e1#ipA0~sDQJ+gBgd%Y7Q&Kx zj@wlxfbvhJM_?Ww|ipVKyen{C;dYQxL9+|vwLxgBBt0aP%p$Am4?qpMc6E{j{cAZhThwlh{PsRr47T&tVXzSyqixR#& z;gC@*Bk;1M?4#59Zu{TzP1gB@_h}MZ?bi2g71 zkR!lt|D?{Tt<>4u8|kaE<2y>N>MP47E>S<@q>^yfGtl-f~t zS7$Ek+=v5}^HVur3kV1dBNeAG;^zca4QV@VLT5FSV9GDrKzURU6)Dp241P>HOk+-IEIFL5lLRycDw56RNm7$x<68;zP+J-~g=bAB^bG~mc z*WPB9h!cUYcpAS?TZO?dVH9{8@!#X8EHyGq9ev;-Ce^72u^zhhT)kz z?YqJ^a*0J@^Pfg4n3E1*fllAMH?_CO#W6*5M9#M`p&8j(hWRYb|1wA#aMrdXMf4*d z7wUopt>VN#vf<0z8zRjC@X>En!!7 zqE6z}?i*#yLWY?0B1VeH(-91cqRvpMB;PZFK~7-w@cLD={vOth<~(OS$wQ&t~-Y3YIBWEJ(+16#S?6n1ck*7SH}ax zNCkqhFgm5-W19nh1cL(;tx<_DNe=vr#;j$IERdTqoZ0S{6ZI+Pd9O+PxD7Dq{;icV z!%8V5#UTQ ztdyms)LSXfS}7|@xx-5NqfRmF>3kBuY9;>2O4Jr*(7nJ)Ok0V1HU-^tt;B>RTH;H~ zg~fw(ki*;$pPCo_Agx>vt2cazh6(fVde#>AGjW#)aXZr9_}3=vgzQoU&!6296B*GhP{ zgfr$M!T_IUfqTe`otp& zskdCY%__h)6id}DuFQ2yH_5@qLNw<~8Dl-h*saECU7p4t^1?u|U?Q)W%>4`=BiP0w zcMD7A^5@}ek3H&v8ZwoGjgSpl`nAr&?c33uk5`*vBbLIR7~W}n?KSeH+Bx|7N+QOk z3m8Mttapw^R4dP5t2Geq!bOg(;bKw<#}!$bdN7G_WYXLhHh1KFRkzXGeO$Bh3g=#c z$9K|Z_cT;X75L0V9@pnjd`DgRbs|=qsZ<8$#ERd zY&a-wr}jeaA8P11HmZ(|3QHTvoC#Nj<)igB95{bHj;^b&?U7YyTE17nAx2gzy$(=| zNztFw%8iDsh+q+={Xpx!lb_Gkalx%mfHNRL?@pXeUy(|JZHccLX6j%bu`kQ<{RND7 z3vte^aj+{nN_`+2f@ul_b=yZeY_(+|$L=9H=6|en2vF5BB>mSqoa%4(OsQ--U|m9o zhHl{+s@cl#CF!sABpBtpRKiLtm25Lf(ao-nnd) z5OZcv;w0I7kIXqv+T>Oz>!Kso|HIy!fJarN;liB_Iw7GOBpMN^Au&iuGD$ZCWeIeUghMprijJch$5C8HamK;mGIm%4ZoxH*jsYAwv2~0( zLI5TA{i?p|?ld^gz5nz4_dfUkPd}%>^VVM9sj5@;eYKoP)Vm72h4gMZc#E$Rxrot! z`jBjPp{ual;_!YG;dudNk2`CDZzQ#QUGKuB?6X6x71OpF34!3yEUG-@uB)*=-q7pY z-h$%4O03~@Un<-1#+cucV`^I4utl$M*o04Ng8o;*X6*SQ5+N2yP|}7X{8A2opp;t5 z(A!k#NC*0jaBHf&9~_~%blo`-5LyRIXEY2ThzS%=g;n`fSiLEULrdszydY6Rp_dXR zl$>B*Hx*Iz&14Z}GcKd(6p1pbUk)yy8RNb1M881ax({ZJEz|%>DYF(cA5pArBPHDk zR94Sud74-$WIat-y1blJNT}^8^wPBsY56D#f>5!c`WM3&{wnE5(cG$iBw1{Hoa)eI z;;4-dSz1L@vw%*bvSN5Q83Bt!C3Yx68xB=7B*4*waL{|Ys$*gF8P1o){*@+gc$*B!r)*ma!Gtgwcfu4NsdwZQ( zAIZH#T}U#fcx>zHM5bL?JLLA*^4z{7>Ev5xKI?PS+T(E2wT|3rFCJ-z?v4)MdmHK~ zBWWUUE)`ruQ(#)V8IF1r7u*_?(0>}Ef@b;0(d;8H+@aAcjBeuNI@X@M4-VX^$6>_2u{42*ZwRd-7CHMJ=@nwomt_x4y zw|+0?G|F8nrmd6g_czFXE-EC1?pEY=>)G&u!9_Zr!I3l&eHeYA5UK_1zI^+}esY%Y zHGEPaxHo-cc)%A?fPu{&OZ({Qr9MFfnkX+#p2-`n^!M~wR=B6)PrVu(8CJ&49Xj7Z-k=K29 zl9w)I_H}w{O!R?lh3v_oe}a1B6hb-oJVbr)`n>rg{&2{l-J9GDYv^Phn%vZ%CZ%X~ z$+a-07425^6d?n|IC+B@wh>6jb3N?wm_oxp$OCeDHi+6 z*9pfofhH$81;GVEhqvCu3GhSNETEEy@4TE1iH#+LePJld8xs;l-abaAq9{{mAb8~S zTTE!wPtw;9rQ@wXwTlzuR2FY6X-Uh%~%dOg%-0s3e8L7 zgxOFCZG{-ufPEoIC}5n4WTqyQ+qCJ@&n>%;MBlmvZ zo46lAVi7jYSVkxHAD;dAb7x`T;=T+#hRPYmu z5A4!F^D*DO414u}nKBQ3>j8gPz($cAtYOV2(XZHxCnQLx3KAoyPenj#lnWj-BoyPV zhLkk~reA_mfN?W|qFQQsZ`kLiZ}9O!1vMZnwBSOz8TtW|AsNuX3*bo`h#*k%AM`~3MVvpzv}0X^&ZE;2WyZtZ&#$L?nUtsZK^}V4wG0@ z-GkzxpQ&ygJsgX0FT-GmG>rO&z0_(6f&OrGy+Tx0)d7{2IQ^T>6wwj}K{|!a59u%p zf9$yYX!Pc>vnjgVbapr1r)RK!@lL8-VpE1g+tHPH3UX$n{jwt>H&8V;PAEGE^7XUv za3ejSBuVk_kk^G3mY3=#e-uqnoyd!7hWQ zI+8%47MtquIW*W_HA5EZJuzVEF*~~Ao3vhm)L)pzM)3R1;|Ek zyFUrGYEM+MLLadcR=w|1?p_qRd(PERVWi;xq~{_yXq##p9A^Im6uYtUnkao9;+CmM z_?zf4QrU$3xKEv+9cxf8o{Scg32?}QoF#^Kc&?T^q)BUS)f18<+|#wzQAYQ!lUA9OPKnPx1WuARTOgth_i95XjwApT<5w5 zDh5I{ib;jqnbf?TuQ`IrPYg}|K#_bSjX|+F*2@m#gx7dQ*xH=tT*|ON!4PP^{IvA^9?TD6E#JZBiMoqLh+5Uyzj)-bkm<^iU zVzOgZGqk+H`IuqfL-8f~3NMp*(1dq59q-URRm)1f-RLdCkr`SwR0xjB4A8Kv-f#Ede!5!D_oi@abcEUec%O27sj=gfnbbE`2{Hz8HLgN$s4Dz>a zIgV|T3$zQwMhr-fcf#`nQ7%rq7sHy!(7hNiQlgD5cOttGbzFwIwLU!wO)cK9*n`s| z(=uRF1PyoJ&>bjJNb_%p(5J-nSq7N&AGd9YfB7`qHXPkbhWO-RGj8Gnnq+#1#NZbq zy2c?bzo0jOXc6a6LhhKc|8@FWAPN1&j5&V{&RBBpb`-UV#fl9vZaC6^z&MUHr7`7% zZbBV)j2k{5&z>>T*#s8=jog^5 zU!l)IBV~kp^y*@1tcz1$T95a0HjIsOXgf(w)EMuBAiW)v7}`*pXulPlFVSyzKV+76 z5QSVFQwd1TKoqKmM^1g|A>wuJ+v#t?yWWP5AHMin!n#C?IevhctBeFKl{gCVrS~Sz zxq+$*MR6j@gC0RbKCJaA$T>Pp)9_3ci8fXs=NK!T4fpCfk)G99CuYnU6wfD)agpvb zH8G27t@z%1jCwwujYGTQiICJtr%b4pxSOMiwif4?*z05AB6*vwAHWGCvU(Pna(`if1Pp_&?shn&pIJcEVk z1RTLC$A(M0;_zV4{7s4n&D|UNikFZ?tn$p!X2fH{4b5)yT!`_&kHkUJgF0!c14b3l zXW=bE_JW|X)W7j+JT@_OX?u6O`^#-*$JIPUmBNei`s!TbV zOCo4IoSg(cxqlk%vVoRpclPLu5rRJpA=t!NEA&;t4qc?jgVd(5R2GwlExknj^lk7nfI7jV(!+Si^wB|mTIT{{LrsmDj@NBYG&=-?41${L+9n{(IYH|uuocoOp zs!uwjj#1-+nTjRB{hdf`f|Y+ebB{vIoexFa38QJ<@4z`%*3U~k(;CNKlB;0=bTT*V z=+cW#BMj(;H2mY{`(EF}4T70=Zm_LXvY+ zKM#pd%<76iqdxkKdQ4-O5s^#c-LJ5w4)HC$m!?cvaK-{Q)53$P{tl=gu-4~IEC_k^ zO`3Do<)r3kpC4<&|1q_3Y-(t7a5UmuA~|i>Q*u_KvxjrHb~&^DamNtPb>}iXD}>EI)}AfhJ_=HDMPNTZ{4F@+FP#FcT6K{DLmu{ zLBIm|76J>tk6#Q=`z%-MEopLO{jel8H|if4 zc1RW|5`F4EIsnY znM6G9{r*vKmYem>Danz}^x2d{9F2gbb!d)KvGe3k^ua~pS)S_5`ldP&ef&3}FnU%L z8qD_=1c^`pD$f^0eqCjC#orq=)<*PxKFXa@0n7q}(>CasqBrL^wRaB|yOdwIo zaFV2`blA#?Dj^D}sB9#;p^8oO{+deQZ}pBmKuq?-$O-)g?gOduy`AbGP6GG@X9Z)V zA^+ZW57E2P5UV3AHf^C*aHwq%Enve;YL_zUgC#V_?)PbnrgzAHOiukEKlC5_<YF+#3%iq_-;V=MO&Gm zf%bW#a_R@ur(|svUZ)6f;N<#QXJa*8US8MDFhl(Mm6Y)FQ8T(G!Gz=<=qtyp{f=0I zW)D4NY~7^o#iRjzOIoX!?#hX_V4{qV43>AVkMxf)ZNaj=uc9ykz}wm1w8c5ZVtNLf z&9&C!$+n(W4u0kzflAjZG-7zz<2`^~NA;-p4%W?zws=OMM%&jL?GPl14XQDxj^0Kj z&IAD0^1%(Cze`_NN-B-e_d)0EG+w?xkM^rSh+T}1=cXP<4*;n&UB5pU8wdYjs(^B@ zXW-Imt?rIte?g9-cJ<=O-s+Dd`~^fJ{d0&!`DYM`_D>@cpEB;J1pkpzD{k%9ggA|@=1w+W$UiOG#4f>AL+iY2_rgiVYT-$I*zDWDqy$2BTu?Px2 zqKgCt1S1JXcKwNA6v3#j`x)XWMPy6}-pR2@T8#g8dhSP7{rna5+@GHN`-|uqM;TLo zuczl&dX5!*ITlCHasI2xd;mQU@MqAo$aTE`LV8Xh^927n^gNKB2l`F)On)Dn{86Us zacup5)7pbb9^TINW*m~b4Gh{Y8RWSfA1N%#pqhgb9pr)Yx)z~Q6TjMXGOvfYU=w_X zT03EFdbLe!%}w1V#2MZm^U|$3se900F)dv-N!xJe#t`lukMI{GEypAMHxr2x**qTY z&m$D$&n42&KZ!_xe-;s=e=Lz$e;ScE{{=(__(u?l_a_la@DCy~&`&HnF0%P?Dmzc_N7w}!_!)sI}#4{9rtT^?@hg$Q#?A$OTgaQe3f_#!M+wpCm z@w~w>2X_JPTX2`-_TXNQ+lzZW?%(3xi2G68PvHJD?iXaM$71aNmvlLEO!_ zpTzwf?!V#2*rp4wQgl+qo>1t+30WoD#Uc7*e+hJ{ZhV2}X8Q|Bm2{ZIxZhKEpb^~- zeeMow=OJ;&*^<3<&KBkd_M#FUUmAzc+>spP+0kMGq60Ncd(V3y!jZZU4o*AW8m)9- z72Tk8b!6qd!QTB@bni(Bhje`Xo^D~xM`}3 zfmfd9kVGqzJ3gZbPdnGaRM&x~i;lO% zN!y90qk{&LLeCbP_P##-OA#5(RF-GFLKlb+uYGN&`&!LjNv358i;g|N=|6=@g9Pn; zOc^PqkCjC~3c(H5AycqxG1=JAh1Eo8G2V7W?Qt|>PDEClniujV_>V)LmR=6?)6NO0 zO(lkPIK77jDbNu^(@dOg;5KPAgqExCJC(M`J_PU zqze=6J!KWo&2WfEi+&?Ic&NptybLvum!XmXyi(4btWL_+4$*p88UyO4H;~mmbhbgO z)+)z*=)yVC*P>n$CX(8)A*~B*LqN}PHQpH6XnP%N5p!^8Y2&2Vb4*jPc;W2EsVKLt zrYU<MRiAY_+0C&`Pebx3j|)h4^4$ltJ__RX~h0rf)CVPJzJmr=Rn)q`wVrlc%rS^ zQ|ngJ_+eZO3C%mXkjikMD^#C^a8PXa>Q7>dl3&b*=&#Rv5zm7#9_?I`rXPHk;)q8& zl$1$yfAu4eii~|uqKhX`@{vh&lM{}T+)EsMNmI=V5i7}m_tEioqW>K(?}k8m0UF09 z5*Qaj4gz6QD^1LHHpYDY6TW?zrrkIM9d$L?c%*K_lr=G3H6DRy?8j>#4t>=We4{$_ zL?jlo%r6mXjzyZiZ}gHL2{H7Uw(%y$xDalAor$^L-s2}RC*}nB$J3w7F#S0{D{(O- zK-GER4U$qwNrAEhBLF#RoTopxVfu5LqY>pF6*~`iJ}p;7l_D`Um2V-$rb^<_e}?=G zbI|S%M5Ir9TF5W(hG=_Ml4FMM+|*e56l53e$OO(YIVxm-bQ-!A^Wr~YL>+qjSvV=y z*VcCN|h#qP?PKKWWRnEyWX`i=y$x<1}oWjc}(XX}R$Yu2)bJ z8loDqvDHrT>n87ABqYHw8((1qU=_6;Qbqem+HRZor)V6v((B97Ix#KKiK2ja?}Cfj*xuT8kM19U1VQ;?sw+U! zH2d?|&8w)*5%pcy3FHsbx@$~-e>O>Es?inU)a;KsQ+HvN47TAj-9;a4#P>Y9EuU+< zy~k8k-62sukEoy%M^v?esD6aM+^8Sz+Lumk@$RnQBBtrg$?Wyco(V@1B@xan&!Z-9C1Swg z80;LEdMJMFi2odywgcRyQ#8o_C1pRG@`d+}&u1q6rDMDK6sSD#S+Sc$ z{jqCrV(t1p=mof5((l?sgK^QzfPNAjfiMwFet_m-HqJIVwc^9rKC3q|Du;v=rn-M2 zB<;i0*{?^=*W$Xb34jxbrmg9qx9qk`E z1utk~phncL zS4X!_Lz9iuzS~o%q3GW*1>2VGtgfKms+B}h$?N;!t&flM#P&`x?utslS{-q~prf%a z`D(<0i6vDZ$=Si8ZB4}?N*Ef5Sz>ad^xnh~lY}5@Fm!?G5mTbEVFUVoKUV#Yc<_T2 zJV57sw?o^IOo=8wtek~btnuMw${sovmRxNSI{98t9a-$P!XlY{xp+y5xz>6o_sX zBwG*wJ~-GgLvY2h?*x5rtWl83(cC)B??w^8H#eL$^~9Qvt53t4qM84RHGO$Ai+V&Y zl}M02+M@V#{^tSy6u?H55zN)vX2-lA@x4s2{3!Y3it2XJ+1fo0O%;;`vY14fvtF;Q#5Ek%` z`FZr1j&Bcbgh@=RJASlV;mx z%Z#vj3&$GplpbN*Z5u86JaXArCHfVCY2-I=N2kV=b@@ir)$0CI@e-NgL+2^u7eb*_ zq(~-&Y$yL_GdeMki0Kh0b6b0{A}PWC1KLOGT_mG-5q)gZk(^HSik`wy5^IVe&wINU zDm@Dv!L~=*dJ;)%v^!fQcUN|1gturcwy#0I(M~$9S;?jFjS;s2tv#L=Q8?II@dB_E z{iKk+@zmj+H5pia-0dtv!-Q_o_c-azRM&>@ESh4^l_G->QrH@njm2VrVpYZ-V%C|6 zLF~RqITb^=Z_uelC%WrS_@yCwW5O-pi`==i!?~?Lp?_11LvBVRqwxnOtlr7yOG1LObczKYt!Dr-YAiJK8h!v@d0nw#SVUriGvU-rIGeA09y`D z%3Q8zni^vOaYjap2O`in^s?WffT2Vkd}k^Xmw82 zoL!jb)(=IFf`+8bVEVhOg(hoz@l5N%VxjmmtsRF@u4zoheB>2AC#jeIT|E~fjU5;< z^*6oLhA(1^{w%)2rk7kTC74olg$->~PJ?HMsD#5s8>MS-AX_WA7-H+)MGo+FLtS-c z{Zu``{{q&NWuf%j5l6A!^)m6A#>7IAxeNUO3~4^njX(DGevf@lv>IdA!*GK9{@5+^ z1fFzdu9zY7;J$*=KU}20mV-hNLg4a|J`uBH{Vf!H&}{Mvm_6RtY%o@r`I`iz&~Mn0 z;j?SI9Ot2%PC6W7mk$!DO87PyWgXDywvoUa6A}x>UgYaf)mQZxBU`)(S%9ofKa5Z=W z*AC@6;|c0s#AFioCih^;xBm{AlEp}gn5hG4oP*k;z@AnX#EkI@;%-8XNs;O3hsWVq zeP`4=|J2*ty}D?~>lpJ;2Lcl$X|#b>UrrPtIRLF)P0%Ml5*V~iLi=_J*6pEURbPNL zt9Vb(qW>4VW;Gtm1fsW@w&b*BdONND!FvCIcis1}hznz7o(pTGSw@vKs-muk=!@z_ zag@oP=%qu^M5SZ96oX+kN*-#;lzk+1i7IJ01Bs;Fv&6fhxj#vv^9s`)+Ep1oD4ugQ z-HFM*G%Xg@^Dy2PVj}%c-DW&Hq2Q_o&Ft?n+GJZjEOSZ(0klg7ziIuH$Ri!Db9}i+ zJ953gC~VqD6YDJKMN(=DbW-Ni1ax|hw}XYyZI?l~T0mQ$Lav+T@ctaJBzJWyy0g8I z)peko+=_2aL931pxZWzU4e+7!(b9`jR5G%SIw38FPw_mUEdv$<98t$b#{|n(Ol$Me zCg-O9Be&sd>WFCDyEY(&*qQ(V_z%O>w#!amCcB&+Q10NYK{EM2kvn1}jF%Cz~L6Hlc$DHmEpBC*M+89LpN`DPyo4D&46jI?SHve@H{K-K-ruERG+Lf82 zIT?)o6|MlE+o&Cwik97v7J(`ly(4_{K7y3tgda5z0uOvEFqW2bkNCFQ1^Y68CXu$a zkGM!wr#HK%%Wq-{3p7&nhw$#fnGuwTOd-`3C0PGnnxlnBj~Tm_P)R^xMR}KVS}@Jc zA*A;xq|m)Vm$}Dt$*xJ@gxP;2xC&Q-hz%|Au8{WC-brhlYggxb5-w=4$k{ZPA3qjgM}a9;HXdX+_( z{s-3&V(5%df@C&S5USZyec+E}YQNMdmGYjs!yMJ*O96CgNpAsMn>R%ctB;en>!pkVZWSM8{N(j2QuY=g5Qe_XzX+gd3$j%!C_!#q)>XBJIfir@(I@oV53vD7-ldnxy|i{<;vk zIU0MEBwqKBh-}Ce&JD`^TzqiF<0K8_6E z&I8r$V|N@lwh~V#hA)EKoyWiYVdt^^X=CX5*qF2nK}~fwl;)lLzWoL!DBFindf_JV zMsjmv?x;i3?d6M*LsQ+I(l)^*O!p6;P4qzZ4-q>LyuXLyYTNno{!x#@EkYZ0J^5Ml z0+;TcUH-&Bk^Arb3BLzLAwIJi@Q1m3ceGZcbi?4E42e!V6021D8l%Au7pFPh6%kbR~a2ej|&;MY@ z!981J+J)QvZ99(z-g6ToPV5lob7MM*`t#d&p1650%tv*K7|m%tGUo4g{*b>1aqio3 zu2d~t1HTf=Mv;3;Y_$ed7O@9= zyg~YGz<5$}EV&piQr-=UTHm7#bWwN6? z`mPPT>N!mI<2l;TFzLY|C9ZRhI7fcxqVxP5yM16wZ(c%LW6wmH&&E}`!6orI#W<0nwh_-@nMZ;?3L z=4i}*@NI>t#gN*jeem8v+xm{w4{Rg%`7g4spA>D=2HV!(&=1>&o*W-(TK6iPS+Dxc zwC1nm_o@z)_h~$sw)k_izQu-CE>re9{&QvW5Q}eot!Zro9AFop16Q|p_S5zaKxYb5 zDR<;T3iOdP>#gcx6us@@k4tO~lcG&qwiE4VYZ%-Wg=0Kgq>(CEAhTEqyGw=*Pz#6Cn5M zV*unN21t1ZTB%=wQ#n|ybCXyjC+0q@DX~#6jfXHK`?Ih+_VZG#);mDXsdE?TU&;N; zNb_fPcndlp-RN;?ZQA!@XVWgBcS~4=5PB2ugI}!!1KV4B6Q9Dfx2KRBpguhyZ zPZBmHcZdc@3&1HGosAft@`>!!f{wvE=n*3qQ3wfZK^@u$IKLj&wCbfSIbUk%@@P^_ z0dY>~e5o&f8l7P}RN75aX9(RdrcE0Idc%YlJ5h+xkDlHbO^U--NMxv|AFKP3I_!o# zkSiNH#H=5TBL%sbm6MK$F}Y9a2lJgpt#}hQdQC}mykHK_7Yy+(PW!;fvmhrchIF;n z*XL?&sF{}{3ADl*iix1UaN7}#JR{s=Nqk!%W)#pHt`VzN=t-<(sS*8l8gF8Q0qhN| zAEy#Fn&O5Ci=xliLkS@NDd?J0Cg|7!2s8CDjj~5hT@myNsaT!yo2m{fW_Kt2q zZpGjwI%B{829!lFI%DEcBX4hnC)Qfm(uP1i30cQHmLs*bc*YKU`d%zslJmNuXMGL8 zk+t3ZcLetWJ<~|1>>@GJc^d1a@uYRY?qg_@<=AEf;y86|vkUtsVG}nuM$*taV40nFN#cX$a7J=&1jCv3-EG?ii+Qm|p%ewmMEv7gi_y zAwCqp%xLeFlITCoG$hFa3KY%6sT%VKjv72ZT2nXJYB?|f`4?v^&ipk^beyl4j~pb#SW1Z-3F*q?E033@D*2~SCH~z zayys+wHWb&R6+8z2sxk6~=q-;Hh(k0k|q#NKMQ?%vYuS@aDbW_TAHf=k#2kF~NJuBx zgaS%za)ih;DxpQGv5;JY7ro(SV1w81GTwbX!C>$;TpJBBuDKx(-wq=2wj(|>Z82gu zzm7h+&+CYefKenH-6M@6VHCke7>tptXc$Ga5ju)wWQ38CjWD($BQu8lxT7%Y?T_`W zN82-OMsKJ2QX8UI7O}0JQEXxCV-eTd8OavWeJln<`Pe)X<_Mj_YSrV}Fan19=aKX- zNjo8;^5({yk8Fz$0OOHuMgn%T*AX9mWScnvB9Cmd1VDrqwbf97kVdv*XX^cJq081w^!g4URm& zgE(PpIRrnN&0-trqf^*z!ZRi^kUJEN5bv|i+N1(@Hc5m-YgZfc@^pOH=@X07yM;mr zXNs>MNep$FkJyZ6Y~5^nIio)wjq))TkJ0im4v&%YaR448^?B6pqeh{f4VgfupGl6oQ3W%#H;(RGy=PTzZ|}&qktYVcgZiFcJ7Lrt2;V*G zNUPOt(T_nY=Ra>(jF}A4Jg0jPysm!%L;BGBjf6fV)K>(D4w+g{__6*3?2)(u+tHPz zly&_H26AqoMpjrqNtr>LP;Oa=*iA3xL*wpC%8l}2^1Az6Eic}g z^=?(1<}gNKGn{=@18tZaJ~nYch6{J&L0tBZ8e8q?)?6Z??E@z~F0K1diY;x^_JD%B!!+vy*}^ z!?$$+s%*))sY&>JL#VCx=fKy&w2_M5$Bu@GBipdSBqt)ugD^x9Ypk%!@-5v6**bSa16y*b+C^tAc748F=%ZD3gXKfaZ}?cx zvi9~K$&N1ZB&^OVs$PtdGAcT4yhdGHQ3~`OG|3ES)$6Z_hnf`q=DvZ3@UOR1#S}_| z{Ouhz$r8r=KKz%yg6^-U^-Sp9wf*pPuGWebO&Tf+HV{gjIUL{kE$`;O62X_#^!z6! zxu){%x!xX=X`L7EiPj@VwN24Btf^d%J*A@d+IkN-8|HkM(=g{47Prp%Ij4S3FLu?Q z^F6?EGC9fO(SUpfmOsG!gm8)Qfd38r6Z!hFPZ&p4U`J#?aL}_I^1m`HAMCL#jmJq{ z5kE{at;&h$-tL{?hd3e1wB|GPbDUW*Uji*T6A%By!_tH!F+aU&FhpXzd~{cL99og& z!(_WXx!waKuoBUdo4O}Ab)PG$H5Xgj!T*wCczhNfP3zu9rlo*L$uD>Qyea~>%|d@x z`YXWS>RLD?+RW6)v=h_;og0G9U9C$ZBB|cuYIGp_1D!7HaMbFLnW8=X&}PBaPi(!1 zcKl)Gs=8Pzxbg|D4nLuI@lydm<;2#Kqny}gNqmAtbnZO4%CNI%seA*$^dCn04^#RN zQ~D26`VVvZ4|Dntua@aQy!wAG{qKjQf5l;yetMGWrze?ydXnj1aag3E2+~g^H2tRM zdSn(<`tfRFU%@-VJ59m&1n=}>bWY{lG~sQUL*HgCxW8nYc!A14bR}4TwaR9uKMVb( z)1Q_8^60OC{wnd#$TQxVBt1l`x8a>3k^`8|MDHw`-dRr|&!RBK)=JVONwXxICGknp zE=i{#5#1=rVq2P!3LdPlr_|r;XrVdH&TsqSam*a2`rDXz|Jk%akg6$Gwu=(0@=KM~ zvO-nX?M+TC5{toeOfOB1hwN|0KxYF+$wt&LM|@6w-oWj!haDVjnHmSX5?JQcJL0xu zf;}c*ZEw=z;CE6q+4pN8f7WQ&_alEQ+>jv28=Ov}iCa1K*2rM|Tc*ZKckmsrV&w=0 z+%gp+AVa;w$U#fj+c|!YTEH!nqDAP@U`Ct+iKLarMp#jLMO$$e)~dyU+ZD4SiQBQn z_*rN+u4#;q#9mKM)8sw5S$j-tXg?J1gyt5}lpMs@m|6r2H6`?5ND@uS3lNl`*j1w` z*?F=e5|yo@EV7f^4$~dmQBY)YJZ1(V8P#fYG1Jqb=9@h zR>~YEWBlYgd2wZ%h|O_|FW}N0~0N znN1hPiqK4E=b%n(Y)ia=d8O&1+yb(|0gTQ;*4*AbFwBD?&JfEPgi{)wgDfyfhlzzu z%qDaEAkT9YVy?Gy71Zl8=4kiO;P(b6l2M{5g$U5o4VNZ&lnI^(VaGsq1>IX`W8`7M z^caVR;rKqys9$q4N$Jtk#kw-s3Y&~Qt21LD5fKZx$b2SwOCie!S0?WC+7JpaKX-=vN}`rpb-c*X{t$z_DmC|H5N=kSeJw5y`?3+vU)cL_m$TaEY)g z9olOci(%X~>LDS|6N_zp@m1|WKuG%H2i4jdtv31*VQL}*GJ+S+CjOb%8s&?3yW z5E3D%`|1UW)oW&8vz{j@1aCp5VYl6ij{^wSg;6qdkWA}5k8x?uph~+B-z^0yoc*$?*`!7d4`p;Jqj~jgmjM%JyBe2T~kP=|2N~&Ig5JHNDEHwI7`0>`H8&(?i z=kVC;5Mv}+a#Xpp6LYCNb(tQ**eh7#L@9~Tj18{j4BzTwR8}x_Zm?VAJLLix2qs?z z)AnGWV6X;L**w?BMx4baW4^Di1qAjaK0%U}_q9L=f-Q>rSYS$%aPI13TDzjs;2t1% zh|7S@Dg2>ggiYL}-`_%o6OwyvEtN!-@_%paYZ+wP`M*;(s0CkXt^$OXepA*rgNFz#`I36 zk1_o_(+;NpVjBH`4DVc~W0^Xb-oUh+>2jtUm_E+*Ri^JV{fVh@qYVE{rpZh%VLG1a zHB1YbRx(|~^nRvKF#QYDHl`mk{hDbH)943f{AQ-fOvf^v%ruYbT&AT=Z)3WF>0?Y= zn7+&OYoi#XS$y0BTQdpx{IkQPi`(B3R{O6#OEbI z4@z7RMwR=5FQvW0`jEa@nOoSrgQ@i;=`Ola()>S3y7}YN`yZXp?%tQQ=84n0Q}H!F z9U=4ajHLL6^toP4?(4rfPP#*%S6rJJQ;SYlOPGBd<5+2J=wOU#YvR(wa1Y~nwl@xt z@hSg9824j)OPGCn7`BGtJjN=$`HWS1N*NE}@N2^SuVKvAhC0Ud;i9XFu?l}9V_8cK zhKCtra4fFoF#LF!`%Pi?&xYa6VYr2{dOthD?AsWt^t>5n-yVkF4|CrUhCdC%os6S6 ze_w>z>tXooFx<^p@#o)R_B~d>A%|`5zL7lfv+@Fl-6K z$zeDp43B24_>~@p$A-Dj48!BYu$8f@FL`0OfUzo{rHoa1sbM^X-(ORheKX_1Y~R9o z7~>Ac=+cO*C(J%6LB1blpUGJH&u2W6{ns)c%D9=ag>f6>5sY=l7ce#tl<_J5>5QR2 z6_=H<;(rcfjJCv;7v{czu}WVZW7QrtF;@IdY%>^{jP)1S=P z%2?T#GEQatjf_?JEsQN}-^uuV#zwOYKZS87W13V+S1DtfT0vJmW7R%yWIRY(8#Xgm z?Pqv<=?in;!B~ag!&sFs^B|c%#h-M>XLEev;fJ?x;q6m+{SI#r!rQ0t@}I}?CvyDr z=FKmxE?%J6XP7syXhCsN8E9o;bx~m%I|HtEF922s^X9or7m@=um^ZJau-YApmlPMd zE2@+U!Yxx^-n`<%`3q&p2zTki68F4I1L=|YvM`(xz?JT*E5;%{RmIirs)a>v6^^I8 zux#Ni<;4qxfwHBHS;~u-vUh`JL2+5JyVx)R1{1Mra{PFM;XFJ~fWdjVZ@fvmtf(xm zDn$ATrNJGQr&JWuv^3oGjGO9c+Qf?uV?e1qBPhdogaPM!1m&yO7 zP$-;*Rc;SBv0!0!Wm#cS@#5lgw`Fl*rOIbIpFUtJ|EgTe7gDUCp`c=M z`9ceoq3VTHDhh&3k~JmtrBeFB?QguPkBfrEh2@2}6fgKSriQyaEmJAJAYV`6T#);H z+$sKt@j28r+%H&GUbq-lV`2G1_d>KBw-r}eN-C->s$3}hLU(b^LU%~<4|Rob0)^*4 zMi+4RUnP$6eF`6cCHy}1-CbH$T(}^l+8b1RW;y?S3xW^gqKbKiOA8lrGa4`-KY>~? zw4Jliu2^!4tBa}@RtB2~gJlvm8`))r)zu-kmO{5heo8{^ZB=W!*7A4F$J8)b zes>EEm4=0Au@@Vv@xK($<)B4|3PY8l7*8cInUAL`VP9sr6lU@k**E-}|7ycBgji%K zg+J;0GCZYA&-h;?!Y;>unIXfVzG~QWu}`|BbL#U?E~~bc;avs-zyq64t|~4T#*C*F zyRR!O^GGwYVDp)w<}_{~^V~o&&#pp&M}?mW?k=n-x7U!@z=h9Mtr}|t4Dq1)nNWQ8-}7Is56NkaJ#!Y54 z9vaLynV*N7%rD1H*v2@UaULl7nZf2WnO+Y{;m^TMaZ_2Ka2DgX;I70y40jc7N^31{ zO6zK-Ye31|i<`o!!%hBCj|>;#euVKKKq+014%(|^l_$}nfjQvG3{Xb1yh}AH&eqfng1lF5zOQi#_3Em znOd1TnC3CfXIj9tl4&i|CZ?O1`k1ye)tPoP?O|#d&hLk*nQ0PJ3)2*)=}a@3TA4bS z<} zzo--xVJxKf2VoSGV}yg$!rXIcz!XpM|2_l3POk@)R^K6V4KTe$iq#S z%Qa@ff-%cPxP%wr8f|gmve3gaTx4u;IIdc}_$vAm%4?>VcFo=)Br(UpfNv0~kpqY%9 zTn5d^-Bg56>7I;`=YzU{iHA0{@I(ny3a4^DiPF2Uf!+c-6aMI1Fde-I%GWjE!xGRO znA4k;;d?}=6poCi3ce{GnRgg-ywo3;myCa*h>J{Q{NnWs2QnQ&|1v!u@Ie*`nJza{ zv6#mRGF+L?ih!9+Z@4R!{=Vimq>`>Yru6K>Zwz?40RPKE@NVcFwRfzmZ&rvANt+HzklS>=0804$H)Kl#FLwzdit4X|NPwZn_u|LUtj#&mX}(#zWmCoul@aX zU+cE*J9h5c-L_}%8*je#_B-wS-hJ==4?g^; zHEqYz5xgeEip(CY=-(UH?sZp*f~B6F2+Tg5W!$lD33PRj52YRL(vh z_QVmoDCJ59N%-{qBhyN>7Zq?zlKM1ol6ZEyRz#L;^}bXZ`sP#0(I`G!Rq^D7WyQA2 zh1IreX66bRoEhd(xQRNmU4@l1g-%vT;sfTSzAe56Dvntd&g|SAI5nJs+g4ebS5daG zXc?scEKHqMT(x*%IrO|0<*s6PX$1`z16Kq;VKW>nJ(CQKE^JC8FvUk#J7b8&#MQw# zhOy3A%@gTnOfp5fdKi;zQC#C>_$0fd%gC5ydvuu@lPr#|B*r9*qszjWWO#I?FditO zA)T?(OJp)8p3-GyOtL?^9E=A`XvkwsJf$n2@em0O1&q}^iBiTSgQTmHG09-*s$qP# zgoawiY932HV>M5ti80AO>DtH``Ui0}Gd`E`CdO)>&}POKwr^p49%CQl^BFfK%l9#o zaU0twGj3-*ig5>Hn&UuMC*unxH0X>;21{2rW0I-T)x$VdLWAKd8UJX;M#gE3&5Xw| zPGWooV+-Re8K*GLWSq`;9OF#JQj3Z5!1!vmcQCdx&SPw2oXV?hH)%&(2H znQ=B_3*#KdnT+j>9gHV4&SyM@aVg_m#x;zmGOlMljqygtPR5%UU&FYC@pQ&*jOnml zx;hxoV5~E~j&Tp;>lqubmifPdaT4QV#wm;!G0tRM#@NAlHRF875!?YOWgN{|>Gk?E zu3`IF#`TQj7;j`8&v+AKGvgMX2!9MH#3f7>|-3yxSjD(#+{6_822z<#Mo$) z`HSEVS`y=E#wm>ZGtOij%h9p!_qgRQ?&)D)*zM`zGa{akFyIc(Zb!F5UZ-d&cd`J>yR0eynug zt=uy<XF*pvhg;6Lk}($43VV`L(M98Ux=2n% zS2fE&s|~+lOj<^4Rt6W6N_Qknp=%*>L>J8wq^mlR9~n=X&{ER87KuwZol7`>rI5qW zRS1sJRm|yG%;_iu2k2VF{z!^NS1J3a`BZc*V)u(UylO~?=<;y5ZjQf_!?}gyA!!O- zBnhHx5xKT3~sPvwyKr}9tbF%8^Q;a7v(WUu%^dMEQ2Xkz@Tk3@_oBip68W5h5+yHcM?&ADY)^tA%65fH+#KOSwl5T-EMdPveuy)Z z_{~VEb@06|7Vqhr0H0-hlxw(_p`|?Ys*B6WH}4ge_75Z1^69WZbJPj zzS)s>x+Guhf$}H6+m1kel6>q`IH@~f7PO#$b5#TL#98NPnE7b z=CDkcGf;12x@HE_DbwW)Ne7j;V7xM2Qv&H$=~C}Ywzrc)(zzIYh+xf?@#h4}kBr|H zXa`jM!S;~i50*C>e|Dg~lko@BOS(2Hy*B)Zwr|zYGEh3yo0aKs1=1<`pBpHLGM>DE zy^P1jZ6Rb}%o}57bL&NR+UKfj&pl|x8qbA8qhC4L` z6CbV%HIPpk z->g76lJ0GR_bS~xL()UB-w>GBE8VNwrs~_Y5PNc;7idRRe$;nfhHneWKXGSvpk7G( zNn!r;!tAdNln)ucYBh-;6yD6hcUxjdNI4+;YeK&Jg!2OV5BBrrm!8T+=%wVws@=D+ zTzE7)vdom@BH7!<;s(|esW8&nzJm>{j6Y$V$M^-t1&o^+S2DhfaV_I_7&kHgfN?Y9 zcNuSH{1js!f5WBlGJb#_4STJH`&i zix?}t&3TLq*uF^FGk+r(SF*k82i7uP!S+pzUt+BEfJ#reiS1SWQF_2=cHhGGs(;hQ zcpcj-Jz)&v4z^eAw9*?Yy`s+ckFon6#=m83oGtmQ+T|q1e`Wg=#?LTTdJffJ%w&6w z?G3D_>d)B0_DWA9P3mT?N(?_-?F_%p^1#vd`xXS|njDdWE} zu3_BDxSsL*j5jiVobe{ce`egm_;bc>jPGIG!MKyL&iEC^J&fOGY`k9b;{f9%#`_tk zF#ZSQOvWEGb}-(>Sm{YeGR|lFV#a2!-%2l9%JyY!uk@xDFs@_3S^1v4icOXPnPi^$*h7eFEE; zvi)?%297UP4FeK#;p zVmwp%XZM#ePGNf~CBq(u9R4V_&t&^s7^`^*$&4LrFQsUh$HDkcwol^tQW=-B{bI&7 zjJ=FEvHJ;(>)HMe#v9o_OJTOJV{B#nbjF+5{`ZVq7)$9L=HW2*uzd&P*^KkYNq!Dy zth4?7jC&Z*Vr;xYx_^Li0sB9LaT41%GOpzGUcoqp?N>4`W&cAMXR`fu%0JsDGIp^2 z^^EfwZ)9A`cmv}a#*Z?tXS_+-bNFX6-pKa(jBDBcEXJGIUP=|9$GuAOV?5ipu>D%b zZH%8}+`)JWW1aECjGH*Vvl;iW{f&&<+5UXS#v5h)3m7Lc{yXCo#@iWZGTzMC!MKHS zKI1nSmonbXcq7L@lyME)&tDj5o9U;fzb!exAY{-{p*J*nT-gpn;54s{sZGo#(!b#VEi=We8!U* zmolzoEakH_^I2YUyjzHIYbc%%DQ_rVz*-MA-Y#bCl)?*HyQA=}tW{BX5o=ErE`zi` zH2gA1;zBXWJVJ2=q!6L_H(~Z9Ee*9NX=f-_(ry+1vXK0e)Kf|Yi$dgq(nS83!Cr+x zdZ{#|UWG69FiKlTdKr>G(EMn5Nx9T~gg~>EgW*Vh4M_*&CG|B$0X|b&(!_jUX)pCR z3n1-Qn9@Pvh2r2kJSk73|4@6H_1+h&@1T+gFJ`Ti)Cb)XkUmJcO(CSj@{)LdAiWYR zZKaY66$Re2w5R#Yq5c;J(kJb24ZLTG%frI243tZ0ue5L~KAI&Q8h)^xOZT@0%Au6Y zQ0Z6hNe)A^t3z=S-dSH9Zr|5G%_0wVFV|9%iHtA!z9qgT@ctw&4U`*+7h|q>Xn5s; z_b2U@_ECjj8NyG>k6a5HjK4BaZl!;cj)#Wt3A0}k5+2Fhi$gH!KPVJ=NqyA906(Sv zP-&e>?@anSrEQgZXyUgjpHj{mEI$gXc1U9SSMiY?wk%K{rTt>)%jG5YA;I)XeNC`l zO8sSUovPGBh1yg2RBz=a<=aZDP5NTWztWx)ruteKD8Eu4sI=KqzfSFf;+NES1naBB zO8YML?`e=+t9D!J(QXOwOX|HA1?q>?F9p-D^gqgka2i?<#XqShTNG%=rG8Oq)k(ij z`bVX0r~M;HPZz9@QctL~>SRy)ClwoEY6t0GUQ)jmYA@?s(4O$(K>3q;K>C-L)DNrK z2Xa0_X!())A~j2a>?!_G_Y{7xJ&^jmV0xs!Su52blE_8pR z0Q4^}sXtb;Uud2Py$@O~FE2S?pfd3NmZeAjhw>-XJT$*jzrO^jkeAfM2h$_<2GpL& z7AA;g4N_S7m$)>L9*Jp1j=UtU4zv#vy93`(srL`Y|4aKCb=(zUP-Dv&4jZ#`djpT+ zp?g(d+!JPR3|RN7jx(D?1Z?Dk+>F}P*R(qf2ZJ*V{O1LUx4*If>QE0gA^7t9ev@+L zQ=2A+*yo**^IpLHsEPCcarL)Rq4wm|8sZ+0-2wN~yfcWyGawuk1nmV5x}Pns55nx* z1Nfl_>Dm*((*4c=7Qyx0%U!_lAFk;qSt*IpCQ$17D>KzMMrV_*=mJ z`@3F7`WgcG(QCi!KQecI0C)G^)Hb$ud;n886s8J~*cFLe6@zSO7D?wi8yZC=qhhyB zkGysxg;V+d^NWZU+_%+Dw0X(Bw+reTyP9a8@ppAZeZM)mj%aPu%^K0}TVJ`8XtOu& zw?sSt_P4u<=1re>FHytH{`V7Y-~Q(Zh;~0d<3Xa9V?RDbv~v3c4-;+fm-z_M%{xDQ zRN#o0n~7Qutos8|U*zn^1ij_FKN4*k^o^jk5iO4s?#x{CC!*%9)1Dw|^$vNGsBhz^ zg0>(3<0gThzV#`h?N6mYO%&-6)bP;D&k$~ET=^_f^9Qy+6K(!Y^m9Zj7wi&r^KIs^rOUMBPAdG&%;wtp_@=AW*3 zh0NQ(XcRQP;R`{nUtRSonO7QrE2#O4F9mI0G5$3&ul?Y+f*QX5QqbnstiO}_=IfdS zZGPqpL4Cf=*M%`*6>Ya}X2|1r-DZ5djfFx#Bx^Gp)9A~OJZ3td?m{ZZX9M=WOI+@ ztjU!udkanl*!E&61@oTRUxF zSu|lk%ijCWupIE>ruew(bC&ZgN?6u(?THo}8p$fsXKudFYqCs^=6l}SKCb6> zXLajNA|^qdK3yT-F{dlqNn;X_Tbcd|@nwn#p(XJ&&R94Su zOj{h=Fr24Y>l_h!?yB>FHnio|Gsi6X7Ie3eM-%g#R5@pMk{KA_qDu!fOrLs@qB+erjWD8L+-pF=}aF@dURamMBgPsXK{S==q`)g3peeOr#C++Z(2D-lh(c6zxAhQ zP3V&^cAP96tVXYA!tS@9(~R!a$??mhLngGW_Kv_wlN-@%HQsnmm}^Nl8@xlsoy7V3`==YwE7uHP-J;za=h0tM)LsTLv`$dJd+MvKXw{A*^%t~iL#GWN?=Umk zhTgehYp>BmooK^^^oOHcikyoK96S8pvkTp1UTpH~U|m{o)a>HI`L^_^gmLAW8P4>f zta&0QkIr=KL0jIdKXRgvbm9+wbfzQs+Lrc?pF=L6ba1oZ=XTsh({YQZwY-qvNI#$c*yo(OGre=rbm5hp_H?Fl zUP1O$C;GTakWxdBOfxGikbFY4cL8M)vedjcorJ zMpkt0;m)2hODySUgXp9sat^ee%-XlDi`&ym1JVbV`*x;1&J3uGK4njr1(s;Eylg^e z`#Rq?e%g^9P%SbeLudVm9c|9A7rR;t#t@JYAD`HhO5Hw=EO9;9hXYyEDc zG2W^*eZ5lC;2vG%JngF2q@E$I=oZ^kJ;x7~ql2S%-)h&s1+7}%`QG+3o#^V+u_LE8 zF{cAPpPnDt@2#`A|8<~eCakmQ-K7(q=KMyO`?@W?vqfXkxm|7PQ9~ca&1r5;FZMfH z7;TI%A6)4hGiRU3xy`dc6{9H(z4MsDnPmo*&RY&%HjFT*=|##dJyn9tX@AS?Q@Sd( z&fS#^U%C5rq^Cv7S6aVxqB}?26FpgBN~i6bzkAvhd-}qr=2T=4JKE@GeMSRq4-)t49jV@dvL$3sN19IMN#?wz+I;Z%Omtepul! zwFUiB(Rj_pD^9fc&EwtVC3OKz6pA?C`{uI=e|bt z^CgM#mjXUI_vn{;bJD14=jbP+^($;U(PO62rJ4uZ(~UHPk3`MvOdHJInIGo+*?DZ6 zc}>ITc7z|1EkhR9I4`3Z`ALIpX}z`D{8xJR+<9bA_Y6>+xNB;v z&^11h^By{PpqspDIqUX&^wURUre+LoL(eg@8qxcK8hxgteB0vOVrRQ9mNP|NyVJva zRBT-JzAgPCpy!knuhz6xqT92sYHes<-$shYX1(a4*Y7m88sC%l7I~pkKIll={&Iw-Du0PD*xTa^rM99LB>Nn&>m)6N?%TCLoa?)TNd1lp*4oQPCjy? zJAG(FpD*F7o#?GOBc|kzwxDkmX(XP$;zkSmx}52i*M?qmH$KTL)`iw?EIcFIttq|O zIF0F_krU(W_Rv*E>60;CcJH~8V7?XYK3jH9MN=<&z~)a+ zeBRUa@sJmk(inGceCSOFJnJ+)JIaF=WvdqN4CzkC6?*LaqSc;0e zEn{4nW>lG0%j|!UwM{dqmKoc%XJ$w@>?^nGR$*GpC>kY;PASwf{B-xq^KWXHyh7ey zg)248JKm9`)EzZU=t0re4hw6T7v9Tq4dQB;xIcIo9S7Dhu?7zl7dqB3&DjksT=Xd`2!dz;W^^kYz3-ciDa6-V2FU;(e=B-LHzA$!{hx`tV`@-yriBH|;|AlFG z%N75BVGM>=ueUV&!o=l8&EK#3g*kV!n}1YEHKVdhX4Zha)y%8Qyrq#Rs+omPhs*8F zsb;#){ye1hoNC7K{8;>7%}n=HyRmpsH6yA#&703apJml~f9q;Sn3pD_pkB?)nK%9Y zl=9EaNBPCGEFOJkCeSlOzMT2Y0g)2dd_qWO}~%pBYM&JhzoGji`wJbxDQ znd#f~^}e(|pP6g&cYAtt`pn!*jc~Cy{>C4ZO%tOwZY23a$e`F$OuV z`2Q1=Wq5W*Tgy)j-z%x=W@G5Hc=-PlbI(?DNqS)w(=KdGleNEBG5mEe`rP=fin*`Z z|EAwL zt7KNs;#K6WsAPKX+A8{EdL?r>Bk!15TqUD7JrMs_GCK0(f1~WLVq{ z_m9lH8rQQTyN}GV$K6xRO+GS8$*UvBH~Pp3(p3C~{EtkNfRe@mB^Av1PHtm9KCNH^ zFQm2JaHE205^<_=#Hk9#*F6%0oy!@GtJr?44SX$$-v!zzSx4Pp4^peJYsX7Ak`;bgf_>=gmGez_x;M8Oa;>sZ|BDct?k@y*d?) zeB(g@X>t`zb-Olp!4>7qE8YA>3=nJrgZ6$h>& zFJpSY-)e2LU^&68FkiO9_hWELYQP`Q=b^iNO zhQIm6G0SJA%(?h23-8@7WnOKakhA1sDbv4At9t>*OBo%f-Hn`fmohCcBsrMmmNG#p zdX|PON||)UQ8!!8!*PeSMh!Dj<`0glsPrt;`R3YNbp7 z|4iEpyiz8t{kr2*D@quz8#U|tyenarrF!L=3QL$Q?{+-p+hEJI-k&d)Fj}34q*t9N zVO%VD+hzBaFmk6#GYmJEFqzsf+PG$xFuAgahQ=%^VH)k4sl97v3Df)9I)_gaOPG#k zX@|UHOBgfWo#ee?B}{Qfs-WfY5@yf1g_G9!mM|G6p7_6n85+=J*+II5d2(#!_yKk$ zOk3@&mb|tl%=phO(heGxFfaRiq$cW?F!%C8RePwEFxtjE{9nQdj809I{aDQOI+W6- ztgx7gJH$ABcvj3D?`aiUez%xumX^I;{(3PpSZG+?@@z5Frl`A1&!ff6^CKmp$-9f0 zr4tMS4{t1HGPQ*Iyv$-|b@tp;-^Inu1B(SycFitkwDr=o^wNr%sdt*{&mLFIB*u1~ zB_CDH@NBBR=LHutYG&~xnhz~zF5dpK{&2ry#_6iE`UsC=#-xYcJawj+(YnwP{}(gc z@``57u`Fh6t%toD)T)?i?Xgkc+5pGdt(ZyG~5WSj2&R8%fv7OZ|b zro2$ZXa!8}FaJWsEbTsjlko!)Lacm~5eJ^1z;fe*Rhqg!30qvPBH z1>{>{&VD}q@rym_Bkob;Kycj=KSAVxkeFEf$v9^llF}zIvUgW!hYp=>xUysM2`YhA zu;lY@)WGnl;OKF&#;o2rBqm1uX*9~+$EUB4qp?|CdNbp3f%rC0bR52ugue$bRTWnB z?c3dNpsSC&fA8?1nCRH(QE~nf(|!E>-Qr_n@O>`MHGllEJMuZ8z_{)afuXVfVi|wU zeGFXZ4F4T_zF~ne_!IWuY=6g|UtmlqzP%%kYkxs#Oki+``*?gvE<7YED8w@=7@yt2 zr;J$FJ;wQjjCBheGa#_8Dt|jZ5}U*j`H0Xs&WG56;c;R8qhjL)g6NpIkYGRW-hE@- zy!(@MzJq=J+b#Bu%ORD{LW&KbN=!sI=nClH*0?DGl&=CA! zBS&1Mj(bbVBnz=2*~m44$SweDm+Wxc*Hjc`jXYVfgKuI<4u8jdXbwpo&xc&UY?6uC zl30;0pV*h)r1w&h$2f&Y;o2pg2$9kC(Vq{}6CC7T`unN*q7snBhL zY>D%QtP_8S9}k4$2Lj2!O`x+r`V8;{i{xo6>=pp;jVY=c7}kQKHUU+@)YcTW47g*C z{sbhpL4O5K0&jp*78I4y73WwH9+?9jfzE(C&=>Fr!hs3E6ksK=4mb)t2R;Ih@6i_l z4*=gzXFs|`%JoP20UiJk{yjnc!QM~tk`PIRh3E&Q!HSUw`p3kNijS528xrf!Ua0-Y+uQm3;7h3Zh@V(vB?umgEbMk2{}Pbx z8qmYduNV1awbliwCq9Q5XA&7jj#)nw_dSWu#M+t?KqpZCn$E?tT+(-CE5J)IlLEF=EznYdg}R%Hy`S6@8YGR(ixd9!6$Lq}Ggi1T_6BIaDaO*e=n z^!QfC+VlQKhp*gW8}zM?O&0*2CYFp@;UhV5frxc7k6OT!?2{J2hVv}qtBzd~hU*wr zi9_OiCJE6Iago8;C)c85GIjc-q@>`H2tQ>!m8)}1T#$*UE9;N>H@YFLKi1MZKXE#T zZ*<~xF3<_oP|kR<-@6v_@SP0Fz6t3RlwAgo{e~{rFW9N506IOAPTW^)NxGTPN%bE) zqBCB?qwdPprL!kGDhuDHBYja^Bs+F_L(T4+#IR$E@z;HJUY3P%CJ#Suvl_{Ln7BGKGfazrX35D^Ui)_LPC09c93? zrqqS10u@vuOo+CJANu-yN`cpg-M7X*Pe-283D%@^>@+ByJ(>c2V>wFO&JurSzbz%t z<#QqF;2KY!Ps!Wx*)GGA#f40%e~24(0fQo6x_odTt+ zEshkiPd2u`hN_eyVq@s1K^fYqQ-(2mqQ*j(v4}iuPqY3jA zWGIEMGT!27Auc=_ir<)$=Lx>rXUox$r!;~UC=ENXjf#NBmWguxR8XceWoYxYOdkGN zqz?Sm@l!y1q1|jW1-!<}RAWC?soW|xULV}ShBA~`c53_Kj=HKscWMpl*Dj0 z(vc4zlC+Hpvt?CydavdJKc2O1c3m*P3a$O%~>w~xj z!+$@-)()}tL47nZKKSshN!~2vC0oG&@%l%7{Ndr|C`&6t*_#}{z#W&zak43qu?2Z0 zG2HKm`+)wKtDwIO25U)`FG-V#G?M`bu21n4csS>fR)>sfO6cP_S9~-DlJwWHR6zk( z*%V27Z9b(PqvEaAh)1b#b=K1r`@NX^0VGdPnbJc&^-xbe*weGo5orrG1r7PrVlckZ z4oLiID9tb&50y%z&yAzPr{XqQIUMI6K)hgCJmv~o$nhyTj8oPlD2f3zXY<%^Y)bEw zIFfwq*o^w`qh7cqhKO76|2#PKS&q$`}v6#!B2*6w%fglgN0)j;WfI zS^>sXWj+}nSgc`t}SJkEzxe==3cVD1*2`I8O&6ogXE+)-K>n;)}5sW$9pyA>*ySDH*S|FkWj? zjbpx!*W{dcAZ-R1=}D zK*w9+R~h^&gs4Sn1kPn09QD>TH^j(cb>PMkNX)wa-3&~>t0+F zTz4CzJ#^6?I+P(#o6<#mccR9-5c4e z?@<;h3gz=<^CZ_loc5F!JHHTL`z8M6F2grpfj=bmCg)ik?pfJ$Lmh30>m~YxP6p2T zdcGHA;$9J`k<^X!yI?K&ph-1}X(ZATYUHcsiDQiJK%L5>sDyQRz9Pa7wlc`7*xdJdXT|6d9@AFjTD3xFtN(Fs~TzT{{*2<#1FroNfP&^+~7SH+EKBt8~r%7oP zs0h%m{l95f--g?jokxz8^lR}vN>aCGhbd|t5Xsezr-QNXS^X<6@-|hZnj)r6eVT}L zg^lvH@-*t=Q}r7~*`LO99f=Ln4}AFE`nD=K?^@y-Za}GvRE5g7 zLYyW5X~g|lscz2AgZ05+6xl^ixM)RHK^seC>Y46#0~5hOD)IYg}W)aK2&Q3Y!A?06A+COJXyR*d%e+=3Dbr(4WHO zCPJ|7FZ1pA za20y?Aqt+I5N7W+P(ij_ZRoJMxQaO+_o($ZxJJE(UK97A^*8ctjBJwUQ{=vNmkZH+ zyGfqmNvBig$|NcCq->AEeXRp#0h(aRHW~6v+#`|uV96$8kAK~l=q3D&%U{589>+U4 z&g3|qV@baIE5$ZnbL__*@6NF`$K?K-Y&9HfFBR`wa_qvfKgUTNr*oXm@d=Laa{QiS z*=1sT%{jK?*n{IC97l1S%<)`~b2vW6@hy(4IaXXQw%?dz6OOGo?!s{&jsrLra6Fmg zr5tbIcn`-HIKIPiA;*;*%dQZYugS3q$95cd=h&NLe~!aAPU3hj$Jrbo;<$k0LXN9B zR_5B(kYjU>?K$qoaUYI{b1dLEnd8|UXL7uc<8vHK`coX&KP3AT`S_4kufW(ickwGl zf3nYhF)I!q#TpqO7ZU4AQHv?Ji0If5kHDzlh!8yXbc+cIj0@que}&^-?mm6od)eBE zUsz7%_YR4T4GhIQ(ee08roQezk|WW4pO8@ZsQ5^)kVJPJOC$Sz1tC$~ekR$+JFo$P z5%G9+2g`$r#*HnGOEa8ky5ZSrTnIF-R6jNaaSywcYqtTe-ky^3%4L1xBHRK6_@;G; zTXYn8c1nFAx?a)IW8wwf{^;_%68N>&a>2874N#Ro?8j~a(Ju6Si4*FP$Z zt#+{c_#k|o8p#o!IP!t&967l9!^1WtlH3bbE>hGPImySuB*DYeZW+&5?s#-ecf=LG zddGys;yo-SknI;A5|ij15;H0~CNdCjG2QS|mlSUMy>-uEs*{w~FAPx&hLTd2fhWF+ zvG}mIUwCAQYbppiK7#8PxEquq5fq0GUsDSv@sa9Vo;(CPB*rZUMdIsfcyCUk ziOYaYuaLlmkni-Zc;aTRe_*`D3=^*hGFvHXKCe6aGJ6J7OIW#kIKEm- zth4zxu)01tIf!Z?wFB*sIE4qhk&59wrmpZ130F7F9S)F^`?~ig@&lrE^CsN^&t!=l z9f?kg_(X-cQpNQ&11W9U-tiG};awBs@KL>i;lUwpSO7qEl9z~n(RIfuUs8^HR6=-6 zbW|ix1@R~PViAcmY#TLnD24Pl-?+M{jwUupmHZL~A$Z$>J+T?G4@>)a-Rt_t0U{%T zxPKfZdv*O|KiOkLc}u3(*Y5?w!YyKxJ9$a$G4E&N?u+vYzd2`RpAd129EO$n*f8Qx z*Z5JR*d-JOBtHhG*yxCmzVUH_c!U$rxTLf=!C-@5-e=fb`e1Ut}9le`LST9{OYZlh(00`pHpLzx%`Q8&f0h zkG^m7efpbs9}BA}s>39)XA6wP>%+*{Og0JEm)V0q`Nv~p!|Ti>S^m?ixW45_|Pom@Ceg#jO)`;}X2)%DsD3eEjcn3bz$T0;-9kp-)4vRCmX-sCc|x#dXoM@HpxC2FXER= z_ORw>{I}Zc)A8?qON+m8|9xNCx)|84)~2xwB2 z9o7?(^-qM;fB?u-!4`PlEP!kY7B#{eKgeVeaC2ZeWU?Oc4zL>XUGQsQ6J)|XU9l_+ zj>lS@UC@sQ&jv`Ex!^p2 z4gpB|FmN^}=YZQ|tq!qo4ZZ;AAU)wiPET03sW^=u*oTw-z+(VnGXkt3Y zf$RYm0K^XAS)9BN+`<%kq%j8%1V|l|!Os9v)(db2K=M|Cm0OEtvaX^rCwqg(0I!i= z03OT9so*qDJ_ddO5IYaS<(ymzK4^wMfxL&nwE#&&nWHWMkxjs(0peQ(cs(cQfJ?D9 zm730@%D|o5iqmufPv+#Q;7go*1>6kxr=(uS;52}gI~9Bz5Wvn|u%jj73%Lt;6hO)i z17871dcv#Q;aw5Z5Wb3wA&GN7_-1>w9rU-r>ek}Cn&2RSlp73A07$*YgJ*N{TyPFR z$|6kGnGl(9*A7@C3!C2HaR8eaybd5~c7fjk#C{og2&QsG?++eghj&mjAcGA%A}?gZ zMS$u|tnUM#bVNUfJ`dc^8Ec-Pw+2Vj=#$V3z#Cog1P1aha3MhKllM3@Kx9AgQh@ld z96TA5m`eCO6|B%roJJ9x;f}Q-&@TqJ?Jkb9C0M@)#tz~eI0qo{H}-%%fYg_85#U06 z2J3o?ZR&wf0)EivfsJ~K_156KKp6CdyLpNAgwOU8+Yj!IbHA^+|5$=&0i-T-!P@V$%aWh?5B~0Z6%7U?C?Hb{Z~D?+u>D$uq%w01<5FgIkXf zr;%PqMAjG01oW`Ri10GN5^@%}ae(;zwFYkoT%dml?jDFXg6sia43Kl2@Q0D88}yan zMM0<|YCuu~20 zJc^=>=V1H+rvZ1Np9+2sR6~9N-W`g52zej)7O)yJ;frB7zaU=$+lGt#l^s}Nw76c1 z;4J|0XB#+fj95?DCPFORfg=IprvRM6$&0~Ck>b3nU=JdrEPwFrDDio57aSzOcm@B1 z!MQPr59Cc?%Q%z=nQ-Db#1nEdxXE~NxyInmiRfd{cL5Je!nGc90N5%S>$m4(Tmo+g zD9F3O9uviNB-}m)`y>r`Z5qaB$XVb)lW~2AO!yE$>P7e#Clg*c4gC*gEe0!0M;}6Z zL$E7A(%%B_nSpZ_`hDQvW{T^02COzJ?u-<&}m`vCg5F)Q1crif65W<-YP!`f>fmIiZ z+lVj?kTfn}j|`0SNJDr7K>EWbaHqu>hoL_Mc3jGhli*bVY3noKh-G5m2;T?%k^Ujr zb-CDnH!!slz!*+=5Wt=bgaH!IV_?fw;&vyjw;C~r9m4Sd@pC+Q6_5)3YVad~*b#!` z*NA-^4_*uqn^|DHOs)@ulK^68D)=`}J_G&?tcJ~MaJRKMryvuK0myi`7_63!^Amba z@HW5!>w^f-TaWreUH~ovtRaiQ$vI+w2=~dw`3wCZ@VE`)_8Jeq2@wAa!Ob>`Wn*xE zP96lF#mRHQr#SfxxXWho`9(MgAoZ1At3=i?1#E{+luLLLkOw&x9KAyv69HI#C;ANZ zn&7*8a7}_N1jp?~8pwom51`+Wza0yv4)EvHvbL%X92EvVE5B_Cx!~Ma_fB@*1 zgS9WCEXX?G!$271V_?54;x-Kcs}x|3F!bhN-YwJxvMhK2K*|~fjsu7fgl%q%$22?e zRv;hgw}B(?;9df<0POs`xQ+b5r+_l(&wwZ073)*M8}Felpx*^HzK{NfwK;@c0CEi? zd4v-5|fOcmW>>lixMiAunO_+kgv@36nMX z3y_yES&vWB6DI5aNgBdr%|Fo-ChHi9Oqi_gCNg2NCY#8F$r@B56DI3iiAQSD~U{)ta~IfVY0T7$b`wdGa?fvYx;;xn5>=4!FfTLta&4P!eo6I z(G&j6>Dk}X6^qkQ|F#XkwSaAFG^aWk(~(q7a4P=S121Pq2z8qr#e=*HdOr99nB*6N zWx{W4&*caDK@I4MQ31*H%_df){y>k@vEC+DWc@(D>j(P0 zAL#QN&?_MzLg>kFvBF4jxNXoKoO4Q&J*0Vpv5o|t+h&TeZz|bC+cf4=FFJnR*XzZ{ z_j|u?Vc1tCjlgZ=`|>H=Ues;okcfF>03#9pMZBf_VHN+Tf4n;y%8hbs)pCMQ+pY@& zYyxwmI;U+J>$DEY10Dg#W(>A!B`H44-9#I@bndtiTn;=0ht#j< zFfEKT!#l-8e@v1e@{Uc>jw|B(;P1aV8#nLXUXP8;(qOwuD@NH*lBAQcpF=Z!*6)I%Dp!#dUF_Z`)Y zj%&i~$@!B7NY0;=(*{|QQ=Hv68{f1aX&1@ypD}p@P>FFP6_}ieYn2pMb9(mE z5LK`u0rVJDs5Imyg^+6oX=KSb6CgFtLH%E`rTAH@EgETSfpfzvdyIqkl99HJ{)*$1 zd@WQ-{tZ!%UTCM$z-(X(a1nR|e8xEO1zgQR1om#IjLibat~fResLPN3R!#CtfMXwl zo4`I`F%Z8#!l6IL+m35O?d`FD1}I({gde&{ZP3l$I<^b(p%l0U)a6G%?Y}(4t|N5k zfa6lxz7to6+LLtEh*w=576#a~f$lS~Wcpw$`R_7u3St}e_^JJbK5a~i{p<*bK@vMl zg6sy6`mYG-*hZpjh>7h#vme{b+?3eLj_53He-J4%)z4DuO8k8Zzv4f!-+=%BcKf7G zsebK#>i_>2;xm14`-bAv4KWKy4Qa<-zw1YSON^5VeBTHZabVk7QlEN!o}4q%?Mv(> zrCBIo%oYHbfN~B*79>t?tB@7%K;{Pd#J;4R2{$00hwhvHIclPbA?)1X#{MsmUqXHf zCNjZ|*s11*>re*lZ>(QG>HWzA+kF$CL1UT|zggRo>tjfg2@??4l9B9l<=!CmAhD9R zS6{B*|6!ZNg!PlN^JREU3uZu=5mR4J($=?Ef4_eJM;;R6U$u|B{hx0Q*e~b&u0%O| ziHx}f>p!R83w6kU9lP&Z|fpT15^ob4MkY=QR z<`e(87*|D1Yds)hs_6jes-*P!L!yiokk=fjyS_oLe|;>2ZYZ%W*)TsN=T|;wpTzkS z3voHdrxDj&}m z+y5x$5_^&|r5XJKZPB`}KSw-Ea^3-S`K)b@7LM9blhNf;}a6B{DgNGOyQ_{}q` z(UJ2HcE1_(uwQ`k8cWg;ndrVL8+jz-G3>49>`Lax+o1b8e-+=8ghBV7OZUxvH0)#C z{dW9;?$(&8rcK67F(L0hZZ&XRJpPPzREFIvz^~qatXns_6S-Nf+eW9ElCl50@fW(= z1mBfC!|VhZv+E3S0-izt=>CY@f9(X={#`#G*&jWb^nTIf?{@F)WiYUIh()`~Q0YsqBG8VCUDo>TnvX*Cl5Z~I>a z{>vhOX9(&Tf6Rd%KqxQ+*Z}MRG5{|;A87DB$G^L(xGwTgzADg=f8^cYAHYZU+9|$% z)xCR!ULuosh~zrZ_TS+z?2_kFWv?M&)=nit+#;XU#$b$ zj$8+S7(epv3CI5qN|66Pa0FNdOaO)geF5@rxEPSwzlt`M`;|H%Paohoz&@L+AH(|l zq>TD8mFU2)fcpM*fiH%L&rhF~BaIF6ill5yFK_y;DzGPu^sC9c>!0~pcTD_#yrKA! zbrcdC->nV$uHc_2U}uJu|LolGN6C?v%+Dn4MYs}Z$iFTaI}8yYvefRo4S6O>+4+0^ z3)sFS@oC3*g>eeRp7e9#FNqsTp9y@oUg}5rrS-_q`p?Df@A;20R4V?vrOT0)5zmPv z9q}JHxOr^Dv6asABl$n#|5&GuobT}8(f>!j{fZuT{<;1`u3=yM^bg~Yv2eE}{*eFH zJCXlH{86Tq|6Ts;{6`z@mbBB~jsJh7|NNK7|9}1G>&W!K{_{;Ne(C!4zy9-0%>M7M zU*hqn5Pf$d=ARm0z4>zqdDPK<5diUf5}3@rC39*b6CI0xLC!(mFWkMxkFUQmT>_Dg zEYtm$HsoyrVk;QX}ANpV4X*`oQ#&fv@;1KW(AkV7_h)i@i*4ns!KKL36 zu=7~r_dkq3^&iiEjo@1@P`4h6Ja5E4yJkxA?2+u_*&|7lOY%t912Oj{bs(_^{!jW3 zDId=%2g4r`>)Y4$R_uDdKhcRuUgyRe-WcgRAojVy-}awtM{*uD=s$cJOv-P_FQNnLe1OcpyR7d&n|Fpe zd6r$*4k(wL(-G2j`l~X0{^|7#=OF3tf7WN}{6|{yzUyaw3bwwU4=A7feC=P}i2s0O z9QZE&@MQz>hl}%f{esii`Tw){vuz-*TkgN?aJ`H_i$BK57V!Otry>^UHqZ{X8a*(NgI;%(s`i6FNOd9_EZ1yu3WlIBI7#r8rRbfD4$$| zC4M#NKS@KYlSlPBSBQ$O5GOpG0)zn50a^za4b1 zq}vkv|Nj0n-Rz)rnVkF@Ww-n48`}=#btb#vn6)JCL_yM`9l!5Cur1E<_x{Ct{63QT z1MUI7`K=_%jt8Xsk3@b0pXz-6kq0CV@NK{OYA^mr`TmL?{bw8?x&E$@&dbj6e>8p& zzh;RtZ4Zg>L{zvbR$oroZ;0+*o5Ata9 zSSJ;UT?xaN=D^S9Z-rPJ(NU5f<5NeZF9algtv-``f##C*C|~+sUeXW#DwpE@1S7Fo zpWzGU&*ndzKNEi=rkUCUorv5}%*AcNt_7)^Ka+Cce@mbTK<;Oz1LXbguf9JebN>72 zM^5#9uD=hzaR0*2-zC?q`hDp=`V9LU?1ud3@_uvwQooG)`-o|S@2|-?1HYF7{gLK} zzwxYZk8Ova_oe!MMY2!&-#2luZ~OmhAL9nOpUFk68=TjO`-%AekxP?H^88=<_p9ZQ zXCtV`*XJi+*ZuzV`N^-^|0mLk*C|QXD}Bop&i2~c8vH>)!YB<-fmrDGB{ZtaCJS@i|aT8JswIq%r z3X8H3yh_T5dRt%m= zQO1;KH&?&$tBZ~&Ca=Cd@k4D*mkiUvdrWthL|wMmv|J?L>uwACF^c&n0js7iHZf_@ zTlwI@ldH;F7e%gGP*iij^}60o!x|ZXo;}uBNwcwAuf`R-rUuB!4}Cl@BxvZK0bbW% z)f8X3kkP4~?fQjXUS0K?n_1lDRn_QGC)U1N{h=!JgVyPpo3>4zofx}mx$V4u(JOoR ztMbq^jz50RUb~}9yG`=T)vn04R%~s2j=#htG`;7V15K>+Q}2b`buHP;8#H6H+wc(I zjZwe<)~C5skim%IJxcFiEgW`yY5JpiohNAU(;~7TkH{F2y+^zI+OpQ&*E-BE+1#jw zcF$GqTnjIDb5wC?&BLkDNHP! zW-Oo9;_3q3)PAL-d=q2ZtsnDhq-vi>uX`&MbQO*ij`S=o3ExdbLR0pomN)6Cg$dh z^D#ph<5x$QKz4%XA=d;1MOwLJDo&)ci! z=Z=4I;nt%@*0Bp{zJIImo|#t*n;baEe{$-CTKM*{pArp*Cog$)azgq6wFf0*g0qAd zw^H3gOy@ekGdY)86}oBP;<(h_1zn%_?LOOzmz{inx!kk6#mBpjkYh5Ys;JtlT|a&$ zUAOPl=B~!kFM3nwEFD*j4Q-!w=-9T)GaV{I7MAs$o$Xr^GSF)G^Nr8$j+oivnnI~ z>L2@OG(9k4R?&RVBoi&(&0`6+50*N+gu3~jxT<29tMkW}r_Va9Us3aPT!+z*AC@Ig zYEwC4cJ}L&CcQ7*j-O$Xajr*!!dwg2`5tjg=Qito`(kO>BT>@wHi|E%t}dgxU&}nV zZ)ij@A@sOfq?M#^3)#*y)Pc^yeSj6nJee;~jWX+B9Jr6=-}p__%`i;eh5h zAE&grF;i=^%i3igpK@~s-#c;V#_XJn9!7ady5YLof1JNM^33uZ==N4mUdpAn+|!n7 z)SjO6cFy&la-R&-Y6cga^lW?eWxO)aGgsRyRatbptJNx-g$r#r?)jLz|Im8dC#~hv z`FreTm+!0OZ)tqR>;0!}!^*UraiLqkbUjxwEbsR6Q9<_)+}T@vXYI5z>kNH6A3tz? zXXKbo8@I2cXWV%)eVbz53)3dv@iFZ-yH98oQtQ(%ZnXVx?q=lXPG{*2s>Q{4V2tD^jJKNJ=O zHp@#`n6ueM&057=p+6O}d0?S@fj26Rl3n&#k4^!>^e1_E~h@P==W5W)N*S3(6{%-j|rUZa%9e=*G>+@mSn4! zn;sk{yK?20<{jtc#N4txVmRE^qb;Ci>W1Rq!=!t{-(RdV<>)!TBr64+@Otj@Ucc{O-tJ z?_MU^KP+36ylP2&dZADu;l_l5ty8>gyFF=k?Bv}-8Adtz$;*UU8{%rS)*qkM?)t;i zOGDr1?sY!d)!LZ1$=`bZ#X0S}@t@D?da`-cqe;&luOx0BCk$M-GWq_^5fM9W=z*Ia zOqNamHpNcIn9o>z1?x=YML1<(l7PhDOz~+by!>HX34F?dtBc=<)g~c8x?&Q zo=`rP(M93&{grn`cWn+hkBffX>)kSkBZ6(+w{N}$VvPh5TbEI{EyqS=SxSIS*f-S=p`6Lz(I-J{7w zj{wKHDZ!=gvwk<$Z*CV+-Dr$amHJgDd(|VJU$j%7?OwmR%4lFeyFIF_JVvg_xf~nf zZ)H7PW%Z2CqDPsYE%~t}0e4b&Puy7Svrg~I2BU|)Z*Do|*|?{|y)7HA_HBOjNMY$s zFUH5Py5oZn!Yxl1s9zJsicaipe;|5sRhMjCGh3ydw7~YkKxdmx)5``^evXs%jgC*= zv-$R$++($tZ+s8iw)?Dia&~8pyR!NB3eKH)q_(*Cish3o#+^Cc_56#g7kB}mN2HyU zxp|}D+_|CevnD=hs(oE8CoO9KyT8yokchEBQ z$qxBZ$#G-HX)Rx>off!n$c+ONO?N$YH-k-gh^Qy%FEzNaDsF9yu?L zZ&}x)tjpm=UMnwp>rc^$bc{^7R#CeoxM{_m-;1B^X)15(a@S0udFKfRwsewvOn}i& z^CbUGHp`}mG{QP_OFP$4Ysr!RFIj_6>4T#B7>?PMcqijR+T_?@CO%2a) z^t$1-;*89w%oB_CLoJKWp6ck^(qVk#-Ctr)ckF4uab5U?X6-Fcws>`O+VRE@=U(c$ zyrRjn1rg`hX}9@Qyeno>wYFzg;eu-#ng~)`4*i^a>cF@x#r{2S z%+E9JI+yuihoRG6RB^_~eaoVKJ_X}UC-y0rVYTBmJ!Ru3Tp~| z?H?ameP_Fqb@zgV2e0%`S}Se{<+*!2F?^UXwaqqe)Hf+*>1$NDyx+dLq@hl%?uy(@X>#shH zxqmgM>eU8~rm2BT-i@_0-h6JUDb8?Z9!7@%x9%Zof11p)fbwFsxkNZlr8R zx_j~n^U!jcz0-rnJr0E%JlZ{MF`OU9&sQt;HLYfO*2c{l+S_I^QScyv6)c%=5S>$?YXn?LX<+&3|;D0EL) zQXkE>yT62(m~?nrT~l;;&(qxxZk|k_rYu{b?XaLWuNO~m+?CB=+WHhUQ9rV^M~G>! z<8PaFJA0OT<-YKQ!!4PV18WaVTzJMxX6zV?rqPFOTcJKjx0)?bRah{@`f}c@jd$i~ z3^qxZsZ|euG;{Y7yPc2h@`vXc2vhiHJKK%B5O|aKSh0&n^&OG5sOf3nyvr^!VT&`A z2X%HW@DTRSo9}L8EBd5zZ)jQkf}p;KDtnz<6C0CMSi33b=H7(;h5?`3Jcw3E*ytPB z>B-7wuRiqg8}Qq{`To}~Y;NS)Zi->6H6w z`)BWN&Um8mc$rM2QqLJhS6!OzYVJE@(EQ`q3X%$YXE-K}Ia8sO@tAjPL_ziA;f!X) z2-Pln54Ihy9hI={+O?Hw&H0*3mRNs|W;}{7*erP5;!Wl@f72dkDwXavX<8BRIwC6Y zdE~f)6Oo~Pb~Hi4FReagN4z!hS+%Nv=Yok^ZWH>tJy zW?^JQ{x9C}r`Q5C`W7qjB%s^g1Lgd+|P+p%=gVbarg@@?s~Gj#Ql8!98#8 zIE7y?&P;0CF7VU-&6(71hkRZS?5DG4;Z2p$)aR9+IX4$Qnr^@Q%>k#+$?pr@f47c2 zmHX!Ovb1&kytl~gy>}=*IDA~QU5~xZd_&WP!aj_lP`LS7=Edg;_YD$E=Fd||+1a-B z6`W-+~+{eU5ASP+6N!JJZzNQ)}mZ~f?{4_-$|(%WBVH) z?_F?d?$GO6V;@$1zI&pZPpj$a6$1`Dj2XE0Wz}9+zahWfI+@(5mDxbEaY_^7Oy19% zJ^Q+Wj*NV2k^9|~yRY8gnSAj2?$m-yvsT9xU43)9JwIiwMN^HIgPD`H>W?3t3Orsk z@A#urH#4I??0op({;uN>ou z-lJ~|uO835*>o7+di3Ln4Z|l6&K~*F%Ke6uZIrXZ-J0#qM$Ep~)M?a{#{&*b`uM8z zy9%RpOzPVhB{vDV*;?COOM89J>*k)XRJXL4cxG(1%9*1bjupS&d+SN*QF^V@hc&gU ziqEfB7Bx1WU~9a>(c#e+!Tm??A6|TPbpN`{1J31x$BItaB)lKm`DOMyy?#R$UdcYC zxX#FH#Ven?chmc3ly&Xl)u?av#~C&U4th}=YR;O@-LYoQ*+P#YwmN%jA{f5^sE02c zHXPr*@amHjtI~^B?3`$%*tXfJ7avFbwrTK9Z=DvK`Elb8oKLxTYHWm&#oirzbC(P} zIo@mS^c?CmwuUe|nV4_+~N!k>7-2gBA{9PZocE7&zP(??_xR}f-ERbGm)eBi ztQb(JZyY`8Wz^eArN`$+Sjy}5cj~m=&pl(u#Z6}o1FAJeR^4oKdImnRu}e5N_5ppM zU5feU<%2yPcH605a{#{x%>IAuy#-uU-PbTW zGt^L$(lOGAz|ajtgD5Q$A{`^$NC<<7s3;{!2n;>Mkb;O3ij<_FNQk0<(j_T%H$IQg z-~ao)-+jOPyWe+zw+_tA*=N^UYp>dC?{m(=40xw457(MsF8PMr*bU>HWumEl?VMn? z%Hs{1k0Vb%Ze&#|w~xvdBStC4A8#+J1+F!(EfC0qa=L1 z&cm%6-|VQrLyoyrfjwt#+R@BEUNYAjP$BMx3pPrngro(zf4&_i=V~q+kw#* z%32Y_WLL6(W^OzvsJ@$b3v$}8==;RuD7y*2otjD~QDS%T@eW=4m}MhKVLspH_69fY z{l_x=P3=SN%ykcW-9O(CCmJMf3u64}5matFAYuLf^*HkKXtP3&^l~SoUS6;}FE^jd zHWAt8{r+v|tH;xoL$}P#?8W!gw38mt>E@a7X|!|e4ZoduK_0oBbMsN;xZUpL9@Pwl z#+5`=?C83(2MpCHyQMSs4gtB@-hVj<#>%T8hwm;B?&Kg*a=AjRRvkH-eT7~yf3N#O zbvXZS6E-|yEUV>U*CI|~zo5c_rW#CIx61A>=ndpLzS*sqF8Ux{v-lxm?y2y^iaC@| zN&y?VBqV%;YvXBtlI_)EC5xf@jf{CoWeNAYoK$WZLFFft+@{w*rEFX6!Za5`vz~Gr z#%U0`eDmvTS)P?%O|F?b;O}+XIQ`vN(msMR8HYP=k2_ziU*JAMZyc&%BJV(3%GAvo zw4-!Oy03m#D2Y8#KbvGkIddl}f(Px>!rDV-%+pVJqdtOI0LkP}7a2p5dH(zlm$^El zg#!r*N$1wltIlC58eb2X{Q>3rQxC`2 z4JcRXbfqm8Q$wUb$)moj%wIGWZGzkv&`%Mcb~7^Flu3V~niQLt*CO^a19FT;QyaaP+Ig z4x<)Rmziac*Drtv)Mbq^KI2D1C;k0_; ziLv0tR$o9W|AV7L@+AHJi$?ST6Z)D2cORBc97FR28uh(RN>w|~rF9kv5vMM+cwA57 zPZ$rv4)5Nqk3eEOh+_wt3esugm}J}oL&zP|brL4U^P6n7y!ybdkqmYv(}<`2&}sujIoDjHJ8H1#|ghPaF9Y zH;tw;^%p!$>TEfZiDlV>!>`A;+PWWn89fRN6jr9~c2Ti+r;b$P?T`#!qvk>-XO zMOhK+g$OaS4S=vrK^dcF)C9dIXLo(yZFQqfnwK7F0!cw2OulAePGDe-~v&h&0(TFZrNt{JHm9RRX#Vshc%Z|i)u+QZo)=2`A|VYttf z>X$lgKyDLMibljv8dU1vixi7s?)3Srw@vXl9i=r9OA;ePlQ4Y6aH56i=re&Dj* z4}6gG)B$5bDT4{mCc7?YSM&RD^&yhzW_4qoTW^xx(w{ima#bC=P*v}=e@J@GZ8tR? z`0=GNWlIO?nWo%rN&?~vD{XU~4yQdvqkwae>wAfom&yP(X^*0&|W zUutN_oskr03F09H)R>`eIuKn}Qp&^>#Ou8YNcpmke76%$Hj@OOHG?Ekhi~RgwP?WZ z2M1E~LP+Tt_9r|I!1B6`7=pq)h^>Y&!c~4*cN-j)5ofSM`l* z&XAYvEU5S`XS~^S;bdCSh`M{lMuYmyr2DRp{FJ2A`d*xJNe~V0!lY43+?l&HD)b=+ z6&@?1NuO@+=K6@?dR9IOq$^_5KQw24t?|b&MG($AvpY|ZoJ;<>q|S&n6eK2g`fig8 zU#C=9q5t{dG>E3&VjA6F%DtR_NhGcFVId=oXX`y)Q3N~Bo?D1=OuBWkWXQH81VbWW z|0dY^N+{9WuAz5Yw9Q&ZZ)J$P_n_yaRvD#fZt_49HVJtq?GKLp`F_j@_2++$zFFo} z`24Ws)fzXU3iF%(94*)CDgFWmH#SdM!z)S;<~c(cjVv~{SwB_c=2BJtZ!o<{>_K2_{J0)xEZwLUmMl>B<%3jYHX$rH6S6d5ML+yL0y7r(0#B5IZ|IsxN2*w|3rT zwig>T1JTUgT(M^E` zvKS-cu`xa2&v5IgXTyduxdOc2Ng|i&4-*dLR8W467d)`YJCO%V<>_ijwV`-SL8jLm zo)zOp?^&bBf_p?ks@Ejo>-xGX3@mF5Miwti29$RrUeb>Bmq7{y68#mbBCC++5XFqkLa@ClqsZ{*tn0|h8%hasiDDh`w6w3t zNGI+5#MqlUsg>e!|4h*G<^@Km41I!&gAWzuzVC+>XSUF|78gHfGT)?t9w zdy-BctMOe6@v*jqMM3CK?J&=|dlVLW`Zary&LrZlPx4jWS=NpVnw`L+JjFhpo`|;{ zt`U)q$faAhu1g)&8eEY*y53AlHp9@lKEOMEA04({chZrBQRj-*c;K=w`!$QJcv1?O zWz3m+^eLstD1Mmh&taA%vL;2>P`7LkvKjb<)h!M}*NG?h`zP&A+vd_im%@)yW^@)H z&6(PvuxGQ+V^2kPY`Sm=xN=2qdw3O?MKiB?pdMc?gIRP^^XK59!@Em8d(whP!nyHV z&L8pzii-%gESGjay|AF~A`J`^os%L6&AyICP1KS+P;Ml&T`m1E=Zr~^v5wqB*a}~uS4CS zFb3MO{e8pU2(dVNq?k5ZwfLp@hljo`{H;1fK2WA+-=Ok~Obvy zxAf;+?=UWP?j5+}fyr$@a~qN{Lj+>fKE{PGtNSX%rvzWAb98pQXbPouT&mo1`_%Hp zmn->OPNwn;Nk-7isoSR^k=88{*}$7DtR=YIvHgL#r>eCzcDa%iFIkjdnBvjHvGie2 zV!(X-a4p7toPjBX%?JWnSly!>Mbqr6)E~<-#Ss0BUsfSI5qB%ES1IWoQm* ze1~pgYlY`nnV6zB!F~N+iU*Q?ZZf?58kYB;rB=na8orX|yw8QN&#iryf>&Ak9M$jR zXE4vPyDwyWwf;T@h0;$jC|0Y%2@g`u+zCEO~;_{>GrDk9kHtU41C1i3ChSR&h`nFz8|&Qkugw?_Bm#gz}0($3jF zykI5-!VU?sC@e(Qouf6^)#7RTht(tD_m7$>?=UrtZ&&hWC2{Z9bo9oHtbAD1E56X1 z$UDYeXTEq%BQDqHy&D*q?i+PRQcbC7@ z!Z9sT4z1T(wr&R%Z>Pv$h0}iAu7a~kjb6Pc$N7W^;pyS@!{adM2vFGWK&-fk%HU?z zNIo@K{&|gG2FqY_c&ng-9LxBo!9bjBB6rfx2svg8l7;the=+M)a<@Oc*xvr)4(?cB zYM;VZmWk=eQS#G-C~ThHxsF-(L*Ox&5$0<3A=7jvQxt*gcMcmvIBvmE=awCPdz^p7OLmn(d-F7)RUHFN<#vFuBo?5jnzsbKX#pe zEh7vQRtTiN3tr6=f3g=Kb8+~NT3QzEC|QK_7&@4^ObLtzI)&Y*ld# zn^LKDM%$Vv)d}V_)MZi197e~uTy|j96rS$#WC)BTEL;$3C%W4ua(OKbn}w?df$DxxrQS%pwDk{AiO zo}bsQT6e~H@%doh{oIt!HIRKUV31Ny=HIr>(z(88yh*wwJ>f46VuYbBjvv|xCbG{g zACS!Kwa8dWlf0X#vOdF zj%%P!UlJuVvG9JG)1>prWSS<*9lp)osePzw&(Tw_tTm|ZAZ^l7av%^`@FRn7qqO@n{~Vx zefkEbE#c9@Dp2t5d)oudVDX8rU*$g7D*%J}1YI*3y48^rSD=S#JLp-ItzSL;vvi6k z57)D<0PNFrE`LppolC%S-ik>$5llz& z?UAg<4s!b}$(GNukNSD9#qS~IdkZxwyNYB#flj#D_Mf8)a6uD~+_JC58dUni=se7NbU>#*Ur zo?G2Cv2C5?*;$VA*Evs{=RO`Jwbs#Q;N)>{#b3C^vnNDP_uF2ZsSVbEf)uJVT;nN` z={4CKE0)iEvK!=zq25~%Cs#o2=r+cWat6KrwW&m~B_-ClCufylg?pR(?mJHiZ=%wK zq;hSB(m;70@Qli>pW53~b~Yb2u8@az5cFXDY625mjRq1{xgKQ5+x_ParAj z3JhB&U!uf3f9`d9Ye_4q+NEbz?ZH&v3|f<0|Jv>na9EbyIjZ~VMw^3ZRnU7kX3Z@s z&46VtU|EVnt5zM8J|0eXRnxHMqydl~*E(ZB|fv0BuicG`H8|x=y~xO5L)OIAV?0c)w}+VA-P0OLC2q=f?r2HL^>eqV-Cg zD}+N>q|CPe8K*ta{xyNKK{ey7bNUmD3h2;qe#(L7cUvF3GSG2M3K=vGLV_F$8A;XM z_m!3UMr*YL;6ysdsm6Ef@e0770}4AIxX=qeBpz0acP45k@G5MYg|oei({EM^pF{Qe z1*fH44CCNzb&lGQ*Zy}h-H*~84W4b6GXId9KeTafb9h?2mmWIj?$-aN@PUTjq_gFM z?~QcTX=;Z<&#O+re6u*CJK|S8~lC@JJ!5rj&%M;dJ;gcSM)QW zj-tQqb6`IupLAco^ILmWtC2xPXCfWXr0aeJH%}P$Sj+#3M?D)N%z{6p_GHNSM!G7i|g~0rXNIDcuZm>A3+Tg z-M82ut0edl^Cmsk&o-**-O-;iXl)rkA|14lYf61HUl*4dN+x=^(R0&o$Z&S$oq=?> zzfy#eSvee{L>6`MsGx1-wIWj;1QDTHNEf*Z6(z{wA@w)9VwuhF2gAJLsr2hka!%eP z{FX%cf-H$KbuwW~`tys;nDW&1pOg3XYDdh@NK9`Q_9{Bey?CVVWQpz)Jx@mXKr7}h zgb;>Nq$WgA3)n`1bD|5>ktl2=DU2FUk5q-gFfb52LKwCZLg9pDX$S}g1*Q9cWno8E z2t}|>%>9Q1rWeY)M>9}}!60!uNbwDXaA3vJZ*cpTC1k9w7?8O|{-}7;JtLMS} zIig5AAFH8fUtj1YBZP(Xa6?@`ZipjzGKexI(0X6bLiBoN!_j;vT`4E6Q)6rKq>yPB zaW~!7C)V6pAS@}v*CeEbso|Sy9k4-Dn@2|hPJGKcaS5Aao9kj;$3l0{_eh|{|H`il z3T2lby_$};N~)+olqP+1mVdoS?^t@6j{wt3(`63Xb`PO(k7Q&E4KLc(dAs9okYL-} zOD7A~y3`C1)Lk^pWM{nM;cMK&*o&<;83|AQE*dytjjkCj=4!;3%iW^(b=`^#5X3y2 zZoK^iZNn>O{V#$Jz4XmDEY23+W{yH3SRFMXfz3ZncP5ThT=FkMbbs(DQHBj6VOU=+ z;UAB}Qj?VraxTct)4t*Fe={-*!$jrFvflTvm#GCit@!$E2&-OgCnxFmftX;T+VR~( zv8m(E1KA`?ZL^E^+j(xSgKtMPOv2gmloM!=QT>T*u(+{^o-i>8oqkZBMgwQN5Bi5l zD>t7LhS(}&b}22&We{i)Z=p5O)~r^gw9k4=Y({1;)0lsV{m>OPcs>95ajaU~DNEM+ z;ZKGJZt6-Nl&T0ArkoUIrLI)j(vYNEcMqlLyuY8MoQB^X)?uN#BGkZuS;n`rU{77) zH+8vk`^nX9KEVZn%8U!w-_MAs&OHc+V@3f$C#^1^y~+o+bEx5tm^g=J6ax{&cG{sQ z&Hq3xHhR5Sj8SN)!2RkpisvG<2;xO|K3h+TB_L9Ub%Aj4IeGM8$V6tICoJ)?di>6a zEVFSzbS~Z}l#BZQ7636F8^#K;V2G^m2$5FwA(7Cxi7E)0c5|G(xnUA=A&*H?4@tgF zuKpqao!u%UJcyJqfOgTtE3TkF(V>)Cu82kFyJA4w3X3rz2EM7)euNFsjJa3`y}hfp z9bZlcX-#!m&HcGsE66IrtV=bzez{=IWUN^~RUq}$8EQcfeR4uWlJch|$v8cwil&n*{L~W0_zaFcV_ga^()0DwBg6Il z@Yip3=I2N!Lm{2cL#aLlW*v!<<>V05j|W@j`+Y&_wR0>eG4Hx%65g!zdyBJ)_IE6J zxT@C>7!-R^8A;-iZVZbt7q#*Cu2S8&@{A`o2%V^8bR)Tjm3sKo>E$QOlXCC~X9TNh zUs}$l_%a!N-|}Hx8}~frYGt>atuGgt_~z`N&M$Q_QHr3E#!vMiTTBaTeFOT=+!kQ4 zR*Jo4ZBQn}@b+{;!E;mtBW8~8R6X4p*9)D91@vTX7MN4I2U!axMDXSK>(c(L?N1h9 z@mA-nzfOd$J5$SK(oYDn?E`C&rj{>9<-9b9I9@Bbe-@lSxN>F(zG6`OqlEK`Y=p}Gua3Wcfq!2SM<22)a+#$_rll~j@t z4L&u&E``3Q;*|Q4VQ+-ZaQgW66*$ndVBY}SA+*0 zO2HCQcDCTi{B(WH@{?8JC5yNd-{UHH(Ve?Na^7(YI#YTl*HkQ{#068Lyx_3aBk zB|iphbsscQE6c>aI2Dmi3*B6f-30YYUQ%Q0>(q-TcZ>t-Wlrm6opvY{V0WU2uU!c!Fgi{u+i!?gm5M<**q4p&YJBE~iw;!}C`{uf~ihrSCLvh@;MdAX7D? z^`r8|%Sn;RXOGsOwtm)ONuhcUBXMw;ulk&PD)XW>EU_OKXI-ol@SsUIQ9e1Qak1@) zfD=KzTw)RTMTB~m&e`YSw2YyA4V$jcK~{mN(|Egc2)BkM%3%(GK%v>)^`vJCcNO=Ku99++J*ZN9P6?a{?h`0FF0$Nq;Sywv*9 zcB|VDkxM!;O!wzCN^&2O7|8{2bhi93p-(ZxI=&na&C8o3dGl%k>@Yz`6+86O$wb69 zG|A;}Z*XvloE3Cg(0zmq02^8CRIgP$>{^q)!`nKW#P@x;-%d}Icf2>Tg^*aBI`Y$8 zRa1%8wScSw4Mc(bb1Nz?kG?Y@w0O!&{`6#Z6vFv2*#B27qDsSrc_x2$$ivaD!K2gQ zrhaW7Yo$`VUdzGxj_q&OkRf2WC1zjil~h>kfN+Yv`MA6SEX!8b>g%~F!oe$oy$H5h z|Itr`72Fn07LRCSwz~81!O8SnR)}$y`r(lViSD9aWFlW*^!4w^A8|WK@s!0?$BRo~ zsu`i3deFOF$Me!}>_YrT*zV=+RHhJKW?N^K*AjTtXAZ zD?@cTg*-{}sg0UkZY8VIm39N#OKx7CQzc$kGE)L?7eM^M)*AaekR{=sambdSbq z9#JZ5{NUa)m#&F!lIxI)DuRr3*zSd33R9g8lWmiZvsl53dfZC!$}z7n*?h|5*=aW> zDIc>TljV_1QrR(^J4wo10^JKav|wX>4$M7xDNtJZhPK%_`v%Ng;B0XpfyVndOBC3$ zqnV(%E%pPSiSplK+g7=j`kDd`^D#|kh$R>*-{+#XeqF&ebgMP=mIybA%xQ%iEQ|iF zV9us^h+mv{?>V>o$k`mQSwTHrwUlsIes;@`;P<%y|%?zPN#WkoeyEu&O*EXZG0185}VVGi=R@+{X?8?Q2<1fBwIPjBv z|6z6}L=_T)&|Xq4u%~Vc_iiE-bW11_q7aK=@u>eyegK2C3GGa39O|XJ9I!t?K-v=6 z$TTRCdbkcQ_-lmyb7ph0kB@G6_8rPphhJKG%ozn?OW9N+_6d$cQE*5vRx@&EUsQSe z?I?4s`I=LIB|9M&B4^K`aV#!q0(&bJYBVf$k0$D2_^a)<`pTpy+;TCOIlB(Qu2$;E zR@F@##Hkx1DY|l$TWO!lNnwc{7o~Hm4>^N(G+ytK+xgoZ9In)@AJlyotlsciw39J3B#Vl9ge z`~Lh3cnV!hL|RNp#mt4{?DKUTW-yx0qnoc)pgYAm63#smJ2mUoPIBqrSMP)Qit2YA zB{=rcg*VXzQ4ggkj4(R&n9cK|Wnhc{35@{K5ZOHgK-&Gt2e(q}po?WqYKx9c1) z+1!Il80ueM$Jbh%YQE%p@iOz)C}1S}w$&CH0i&Z&a2PdHprc2IM0e`aTR&R}EYTFT zL)^wm)KYyYq+1rc1oIdkue#X>cFaF`A3sz0JVF3WupY?Qqzb&}rOwbaCO{n4QoSGu zCfE;GX}F|FjzQ9259O=Q0`0 zTdRTFwjjEi`Wh8#U|RU=fI^5M)YN}Hjr-({)ldi%cv95Kv$;>+R}F(85Pzjk-d0Tj zK_dT7@q+um{;rSs^`>f~UnvR#N$_h1F@e~CZ%v3XIMam~K=eUc7kq1h^OK{Aa24FZ z2L)%)U*Erf!7XXO4nolSN$#JAJM5&C3B(`b3qeCXAugabG{ha^1X}ilI77TZVVz%P zkl-2PfBi{9?0-FP90lHn4TY$KG9D0bh#e?@9YA9TQcl0RP=&aGGcSPkNf}RwA0z;@ z?EiN^;C=KE22lTZe;S~yAH?Ao9Nxc5ozyuwF8`|G2-*k&B@h5wCy*Zi`b2;{U(lmB z_;UhhC-2TZ!SQ#SX1}ibf%?DO5`&2Rg8kgDx4oU<=>MyXCuqg}59t5Z2e>s4+;0d4 zZwhw;)Hx}2f{i!8^@L(B;7#xTpx+Vzi#TXm93lzM3_;nGb`cOYP{SX5d?Ah?B>>R| zc^AN0AVdNp2Fg1R2RYPkW7PmYsb{6VjN5P66w`11w$JA;oOzy}RV zi-NP?R5>ZH2q6RQ8vG68$;h5S{+li*slP_)U$sO){b2BcK>nhQA*kyGpbh|I;{OL_ zz3{zn!O}ox7K(lM?@R zCqI4_WlAzdJ3l`s4}16P2+)A1pAvt7ucy49gR7H=ou9A=+QHY$&&%0g*ul#~-pnLD5?sIPftr3!4gtPs|LcFJ z!4+R8p8(K{lcS+8IuPyd`(Z{R74{(~o|t|#A`8GmR6$iiYr$Z_RKZ#Sq>#E0QHU%=6&e=W z7J3&(7A6#83#$rS3kM6Q3fBrDMbt%zB4iP&$gs$^$h#=AD4_^jR8`bkG*~oMv{nQu zrY=SlBa2bRhQ+qU-o=r{3B}mrs^Zq-!Q!dnwPFa48i&9kaVVT2&KBp5i^L`1u(&E* zD{c@sgq^hK~WUyqaWUT~JN?nR5MV6vU4NGlH zy-OoY6G~5rK?vyKn(Lh#oU4;>m~WmR3^0$){|gIOfJZREBo^S3R)_=GGyr_M07jz# zr`fCUr4LjYhR3vkf^*qHyqCm3K93vfyUSm6L(4FIz)fZHg*ZVuqL z3owKO9GL)?0sv20fT<3^)f`~!3h)gE7{>yf(@L?WxYDZ9hSJv3u2LWk2qdi8#qwEt8}z7Nh~HHvo!{0&4FTzyX~F0F8AD%?n)t z=f(cwx(2{;qlI&S^P2$RHJu{!BG;neqS&IeB3w~JQCHDu(Ol7P5xkhGSfE(8Sf|*$ z*tIygIJP*g7zenrt9Z0{u6VZ?4tP@l@TCsmNmsy+v49tGfDa)k5EB9P`G2Y$V*x+6 zmJXIqm9CXS%Baf_WymsAnPHi2nRi)aSwb1Mtg5WFY_M#qY^@AZPF;>DN0y_?4a;rI zy~`uZ6UwpWRpqVagXL4@Yvm9;H6DRS;!$`*ye-}vABj)EWARn^R{S7-3crSjR8UtS zDv%YZ3d0K93h#=@ii8SmMO8&>#bCu$#aacVlDZO6iL69b8dlm?dRInPCRAc8t14S7 z2P>y4*D4`~fYv8Q2uqDcV3Al9)(~rp^~OeG6R=op6}A;Sh@HZ&VIjHHxrkh3E-KeB z*B0of$lQcnY;ILXq=pZIQ6InooQ9y+`K!@G36BtgsL@@4!0)VGn0YBjYC#?Zaf&eZ$(ZuFJ z7rO#&91QevEYQd&{PdTe>jHXt6li87&@hoFdgT8<{%=nKO?>YgAzfISBS}kh@phgC>a=(5UY>8fMopR z3Kf!+01Ckp=D{K%NU{G=mk_bWEiApM)yMiw_7yq+MxTnUzFrQO%TbLd8hnzD|4N6A zqD3L2h`f+dgqN`dFenU0t#gq4#P-s|`_jeTckhfmWH@Q_Z%CmO0G?1WawI7+!6ias zI+(Va7%P(bBmt+O)%JC=^LO$^T(t894l-~}eZ{Der%p0SC`e4~+?~CA9mQCXOea^! zC}`BZJiHve5bCOk3uq5CaAd{KAURL+2q>6TRaFt{PQLzVXS4%o6Jg@y8;Ay;DUySc zQd|rvB`$^hby$L=teCh2_?MEClCwmb{X1|>NQPg%(NUQ=IryVpy#DD+7%B9tFFKyz zIfeoD?q~;ux|bv95AEU!rUnROFE4+Dv7|II3dZ#(9l*E{M8T-QIKs$aQ7|av{`ZHp zT!cLSBF7Beu%x^771FoLwcl|k3MqdK|0;$x@+yx`oHo8&_cis?;B5zzTF1T@Csl=zuuGX(s zk(^?6LgMmlv!u-QBUbA}PFkfXg1~S4D$YX>#n3-JY9-5Gh6I01GTdyq^$nVgh)x{{m6z}`T&rc5= zy`_@Q6EP4^@YizFmZA$Hk}z-(X%PO>;#5rN#x&~lvlqVMaVMR+8b6_YZl;u}s)Ga? z;PT01-ob%GIs$#Cxl~ix)6ynB*QzHA8H1c7s6};9QpxtY;7nqFHXxmTdUbwhufFMz zRxW?-s!;vSMIxgfJ{3u%gId{+a@4|Y`Bf38Y`8n+v|O@cdwiLtGgvFkyYwMz6l3}A z`TFit7&g=h1Eb)ii29xQelr9SK8k1kp=#|LBxuBlO#3&KICm(1f-%Qp|m{$ z`k17$qZQITPBLufP&vvU&y?u(?y(vWd0r>>JMeXSH+jVg>7L>4?jvIE{@KSjtuNH4yalyNMT zsv|0ZQu9VgCthK^_3f9?uv0;y$ApAHCS#CPC&lSb(nLrC@Pni~xxhc8s75pK*cAttcPNJ28Zo|SN5oHW(cXNu_r;n9<+82D@7vD(6(-Aa0r7PL zy3?m+z?T>1+-f9Ue+OpTd0@Wp4p})$D-ub`r7#iX^crO@GRWR3Cta|P>bO@KS=@xt zF^(1+7hgI(Q*=SeUC!?v`7Jrb)GZd*FU2~tC2ys9gI?ZeW@wMS;2bXM(4RJk{#>qz zv(D;fv#J>VA?iQSaAW(!%`BQ!`-}=ry77pQ+WVi|jB~ybUz*Q~+mN?6da=YqCHT%f zGI~Zc)N&LVMa%|NF5%xQSKJ)kx76rvXp1(cf zc&cUQ~aqi~&{WHVJm|LNrhx^7D^vZ9wKZg!{jT4~pB@8@=X)mJp2N_~ zvDz4rYy(^(qEMp%q&PJpE7R0_K$45tyNAuF+oWAUcYfAj#QF4h4q^^9U<@M z_=XSB5^?TD6pOBnDF$I0bMMgid9SyfS(gghHK?b(lrm^G#{_%7a=j}i7roA^|B&t) ziT&E3TYBLm!)!K*T#gmHdql2m54KkL5^jzh`P?eb*r23Wmae142L*)me%F$|mSED; z@zCcJiTs(p5pkEP$DXD!SV!#5p5N-s0gQ;vzp1nTRA2o|onge2RD}fkw7OpBb)1!> zUD7F0q5p5x8SRNK`lZg`NMfMQfc^Z>H5$ub8jTJ~^N&U&AtpM}Xaq>TzZ9AVQVog1 zDq|I67TBVvws>rd;}djgs!9u z$INX%U6VT=!JM}&Ry$87d^+pf!^3v;`I--U>kPCPwwn{2o$d+yzbk|9T%#d?2Zc4G zzf?`+@Z2F7`E;dVs%GTLz_jf9XCA%$BmIsvbJV_p!+TTn=kCnw4%a?PB_{i_P1dEF zk!Xz>l@3aK6RG+Lx0|TqkW<=cYN{8xu*K*+{Ltqx`%qhKdY&bQeYhDhq*85cQe4duk9LJb>lyX<-7UX83%oiWGg;<1a&6%zQ!J}k)K+ZmL*CXiqHr{9`oi-IXK9d& znCrw}R8!8rLu`{L&x8e1WU=$P*+fIP&!+cPe(GGCVan6b6g^AfT$B{R*F1)jqVuXK zl3@v%!?PJVkeoFmc3-~1v^~A#=vU+cm%dj?m0l6x#whga;3lNB zm$Qd+%4Ri-Xxr&6ja47z6GPzb-`E!TqZAb;zG=N)6^2OB^+|_k&6e`RiF*56rBCTG z8w@&Rc(k+&nuu{t1Q-Uz3~uaRTX7?1Rldb7^r-7bNV+P;q?Myl${-X38J%q zvJ5$J$^Vz>=@SDXiIf$W{l`Fv%bX;CyW5!LU-K29`iba>C6O|6Qb;i%t;FPi&sY>; zPy#xbuBO=EjyM$^m0Exw+7rZ>5t^Ro-e>DNHjng-o?)ScSSM^DpfZp zFkA5Q^b2tJ2fq359D@HQ=f8B;OOr#J76*Frgr;#tN}`^~0r}=e>d;X(NSnE!SE3Er z(kyb=lw1p8IjxkvjX!sag3S8IN2S?pO`Pl6tvnHMHwDM1+i72qE^_(=-cL2Z^oWIH zgFNlqi#MFf*^mW}nTOhe&x3Z+vK!m&ZX>gNaa89CEEU<9xPo$Si)MFig}7Zj-BBmj z+Ux6UW28ZCceiB_|5}rUt0^K4gRUAHlX0+-j}lS*=_7IW%9ZGw*I%KEHmYN=9-$B8 zqKlsO8_VUn=k+a8^JdVnNZEw;_0=`cyiqkiwQQG`%vxv7C(&CLb4z>=LRRc-=R3x# z>~vGgxJ@WaE9(dLhAWF0`L`_w!w~k1aw<6<7qqXb)_v7RX=U`8r@pL8*-Lo-qfdOd z{l!uArpZ?+9?dlBt*IX$zCBZ|pr+4st!7wrP4=`Ip?>q!^4;V5uEf*UCF7c+G;bZ` zUv`jB*TiKq`F*bLsC@h>c}N+*seDL&5yU$jO zs=Cc5Uw9MkL@WBZNPVlVGyb@cy=aROK zw;WSRqBS^dTgcCe2*ZO+tu^}3%doY@{PY)mLZ>@sp~TDbB7=QvS#fo1Lg12PquHVU zcvy^wv&{qD9XFmMy@$BM^D0U&e_OX2VBISI&AR=k=IdY9&2-SnD%ha7-72Fy0hw84 zfseY$`R~?^BoaySk9C8A#n1oLQ`h~=t^v!1lp?11B`Y;zP#BaLisb#5&457}e%TB^ z;HCdzFGST1O_2Zd%%HlerWtU_{->M4f3@rXFOIcC4xU`pVkh{2A3z z{xinW!R@Pa z^=TuCO=n`6Rz?W^v^JDzw&B&={wT|)`LCTeQm>js>t3&VHy?E{aRl94`J`O&R=#;_ zCFlLSS^oN~z8hidZ|HTN^~gt_-o#WHri#D%fL(L(pnN!LenZ%pvC;TOKsx94T-@}s zThObtxX{WThf^!gUn=fur6SLD5bkWNQq%48J^CvCNxE!?X#ODS2SWu5cMRb-voBqt zmYgxfx1aK+h#0Olyi9nt`7}{TWv>3VM`}zmZ9tQ)fvvL3+xlS!*9mIEhx0YA_1HgrfqV^Kj(s^{KB?l`+8#2s&&@HFd^EW-ekJFSZSaKbWX@D1T3=cR42wA45u;R?Hs z#CDm{h;vlEpINEMWGKEe3D-R5g~XpN(Pa$zkjXz&-IOmR^v$+`m?Ibe zsD&sU>o_Ok?~<_md|B1sSR1hGSFqvn>VbUgZRNV!7&Fr94B-bdoR7TUA2gT576l^D z*sa1GQvdCJ0#%`1_T6SYNix+P!;pW_JPWM1tHTv-BOq8xX zJv{{_s$7-7Mxy5Vp^-zqV41<**Zaod50`o9tnUASEe?l-6-GbjFFkjYDon`*Ha+rW zcDp*tNs}lx2*-1vaU=9N^Z`U_>O#khm}`+1>CjLX7OOWGza3!5KYVW)N=M(zwlJUO zpMVB5J(l2A(P5=H`+-A(h4njiwn1Cap?fhqQQLsF&mF3_My&5$Cd{w>NH`^Oal`+P zxps1H6r&t6ic#txz9JxiibXN1}#%=9Bfq)tO@s8nWmCxzNGNN%Y)0_pAg9v=BaUns4t5=6O}kpTv7~)loFE@mz9*V#75B(04MYX2vQXM?Szt7IKQxL zj7YiNu{TdREkXDv$DcT%4u80EmrTSMk*9$|CnTdgrGa)qpG1_-A#^<*M8p)3awi!C zbmG4=5T+;lBAgr%+L8!k6IDs1oHW8jM^#)(8exj|a5C|?^YA{2#;U9SivNOqap`{? zQVA(BF>$Hi4yoF|13~jEbOwNc|L#i+f#mqBd&b|Fesuyie|UH!PN?u-x{uA*{BpKa z&XlDh8?8LPS>)Hv9VT)zy^dA5mOfqh@{kOPP^WE%zxLM(YPq+F7n{BAp=)j@R3UM1 zFq?cZBt`qOgr34X^~aj4%5XLGzJH*?8-dl_Q!5WVL>775#}CyirCo~!35Ht^ZzRx7 zv7rqny0ICNRmV(?^j&Y!ywT9`fFWHE1K5#M~lWW6tA0`bKBO;~hK zVo|Spo(g7HfQ2 z)5%FtWeB(-M6j;$o>Hj9JlVa@f(zH!e1>O|R{T70P0#co2=0XYJX1W6_6rydrrB?= zNV%A`DC>9i&YjV{tK9DcyZ3UexjXDiI&QKpaL?pozue1nVE<(Av4j4$)O9tRYM5vy zazYut`U>58%|s(OdG*W!F>hct5xh&tdMGulpK@sVnJh|#2H*XL-^VJRyOkxDFOM&1 zvQL|J@_liM?(|g+&DQ7crWBelu9df=r1>gG^rgQ`i_`bslQ|=$Hg9+yGH*4$+;kbv zSy#zAan^nzJihmk|I&gxg8_d^Pih38UN1xXV9#Tk512c?M_KAI(E_u3zum`sz^ zU$}oK%Ma|lCz0DfW3_N3;^aDlkQvGNPmC6ZB>z*aPnJS{$B-Kt#yl?G67>@d1o}Ac z8X`Ws%j=IAa?a&*1VN_+PA51QI$T{bNa75{S|Ke@>T?pBAdQd)*o#=b7~OxxM4i=npU-$MA84E`yC>Z-r|L`=eO2MxqaFgKBy$p3yQ_|Kk@=D+KK zxXOj!@YtOI;`e_FB>(@QjKJay6rzOZ;0#Q+nwE{G)gDt%+@_hx)Q7Xb6)+eW{uqkU zd>Fs-xd zpjo)6@S?|AGlh4=j3m8uB2PGO_sw4osCw0V)7NUhzqnOaVu=tsAGZOsyM0_r zfr9(5zDc}YF3R&>J}UUS{PQRkQ+n8z@$)-pO}~?x#f@n{H1awa2wIJ2y12k-veWK1 zYQ{?{((BPo(vpc(_!en%`e9~32?_5q##Ahp+2m5(LRi&A=$OGW3&n%9J@{ z?dQF_!a*&cn;d7!`Zi6W(KTJVY4zRnse{MNP6xsLo2h-5=AkW)Qz2eOBAG_3TtbDo zERB>!U=B#iT2l=U+^n8^-%T0j&-wZ-oqpzGh$aH%zYs81u6PD+N3j3e-GVUII>VFo z*1gjNx%HL=nszvH3by$SgMi^%34Qlcq>g0$vXsIRT;o#nQ=t*v~k%$RXm zI#F%r#vsju-sl3~a>nP+dJC+cW{}Ndsu*_Jz9ybF{D@Cue_0TVYyApukfoX#FoRW( z>JSDjkVyI;J_=MPRC$D{zA+K;gIFf&Dcg~M%b}EGh>Aff_AB*o`$Zxx^OVOlY*f4aP{8@@RjdENGD!P> zHyx*cP2GU$p!h8wC+15GNGuR6LUKClprD|C6sqx#e{aQ!Vmi%SD#6g5z z;$P-V1|-FQo3H;%Z0LWrme~;_MXX!=u#&y!Rw7gnUY6Fxg$e2-)3LP^T1a1 z-rTd?rpeVft!7gY`@{8T_>aeT$Rn~iv#BIF4e7}F{8Bx3FXLe0hw4wF$7$C6s%uw- zBBSqefv0_qE4^lnC}S2++ttXvZhU8-dz8b~l+PZJwBtpK~K55D;@4vC)UOM`tid%i#8}&J>>&*YR-pxB4u5=H;(JsR3v0@BH z?`ER+GCIj%bp}y0N|Z69264rN1W_}(L6j(qB#5#~h!!P87ZOn~35$@Jggwc|u6>f- zyZ5=zv*-Ww{r5fZ`MtmQoDV*nNJOVph$$@G(Hq~YhB22}3a1N7yu}qKUcjC)7x2Xk zMC7yUa>HN2og+grmj(H6;T7c>@9pj7E-Tq?T%F@#nTitWPHxSgd!<@{K3L(h@k-q` zO=A?b3Q!j+N5co_4G0{b@?F9^eRtE*nwRqGbnfLS7LJjHo3fVeo_2V zb+03!R*jPyyHSqR!7MlyT2Ye-ouZ32do#F(?6HS3wdmwN7{xvuq4Uh39g(^}P;<;T zB)~Uo=(lA3??&XCWIBnijfq!LBYsF1qj?E;wUtJE6$d@ykIV%2o8OQmRM5a~A4b_zf=oavotJ1$F=WmC2{SCmM$SQ!(=g-8tVx7>IQs;v& z)8ZiscN?FCXuhmwlVCW{JOuq=u$7J2ZH!zn6ejw{eRo$F^#RpB1g2b5b?lj>S+cX8 z@WvfPbMs5X+z0BI(tSFOjhw)=R8gg@EoBx%XVCE^Jp6-cC^g+cqJIF(#8EjSPAsJS z+5p_)wPW2~M3P&sSni9silvV?n_UEOUdb&vVZ*+s!!}EsF^EQ+{W6AHh9Ib%k5iz%eokb4FSjN*<7%NrO!;Z;-uce>{1rTrSe)bk zV`q1JyNBP@W9CTe&bqt-9FDIEQKB3m_RHtIAHhPS-S1a_#yb#2C&RaOaxh|?f?@(QK4(HX&)$%2 zNO`ur{e6aYpYN7jEB@3xH}L64baZDWxHUdEZ&MVGsmAAvzE0DqR~$QohuwFgra#?s zb&C10wdm=NnWWYkgB4~v@Qj>1J3(~s^?29^K>5c5#KL?b7F`xw!uw(I zmCYIB1uAtmo0G))RN}b)zr^B$@_d3B88hfNVrih!7=K_C)7Sbne)%$UorB5)gXZT4 zXl|rnFc>g|eQ9p~32^>5X0%Bt*71HVsH6dIeWA!#mAwf2C7Bgi z|8$D|Y(iq>bX^i&{Z{a!pzgYw&IcZB@?;G0lh5ReBq*+2e;cLBkmOr#9;I<^Vulm+?XjD_Vj7NjYn<7%(%7+1;d_zQ#8sDK zsi(1&FI(!D;YDl>#epO*=~HPT;94_b{#dUiI#UzffS&{^>NGHOAr%hsYQ4>)8jMV~ zOAN;Au&zFTRvzm4Fe8kb^9IDBliS48r=~pS2BKqJ+aZVh1A()vEwP{`{Io#E+eMTl z9xVQLs;m~g1UhDxBET%!uX~x_f%Q!uodlNldclvo$^B6Lr>vaS#ZdDx<5hhenx)|qU2v>%z=Vh&sHKX7c4jwyHVMWmAF$MyqVs(EX1L>J z963nI`njtstn%s9oxAWn?X}i#%{_+%%e}CJKD^?);eH;xo;*iP4QAqk;ggC}TXnef zkG4UMaQigANwv(Fs>QgBDZ~73h z+TZ;_5dvEKxYnemfYE>2-=5HVK=k;#Pb8&zjEN{v=Kt1-^T#uJZ-SggOF%%7lqhdy_`ocY{mF9%|Ikjxs#_$!agTrP(Yqz zv3aH2&XQbk8bw}c)wl)bZ~{F;G4A@1A_Wgm%$zw1wlDHr-uU8?^pmK~)s_4zgJc@f z1>o;DE;(go9A(mzF3;|Figi=fJq|rhPT_tWATt+ zy4`0>w>~{wVArKGy;hRy*&S4-Vk3t#Zg>Eah-c9*?5DE55qd>($EE*vYv~K0f16+o zxf|#?0#lTVk-GMRqk#tCY!HhzihOMuJbdVvW%VD3f$Rn+o8@Ec=rBhvYZO-PCl#*m z=i1}uqctaZg$_xB4$QrEmONYFXOSMRJCt#|6idyOIt!W!R8q%9g(vdt__qdHYoMO() zP{*|9+@9%7*CxrtL@>USjlQ*F5W>XD>t|ubs2Urjps)O9Yn>w@sB$Vffv8FeUc zDyp4+{wCG5CR_w;(({gz_k(Mf!|{-JUU;Vfg(b^{%U<@ap$rtn}1EYe*x^_;l2O> literal 0 HcmV?d00001 diff --git a/src/hash.c b/src/hash.c index c5dd18f85bb..d39f68abaad 100644 --- a/src/hash.c +++ b/src/hash.c @@ -2091,7 +2091,7 @@ BOOL IsRevokedBySbat(uint8_t* buf, uint32_t len) // Look for a .sbat section sbat = GetPeSection(buf, ".sbat", &sbat_len); - if (sbat == NULL || sbat < buf || sbat >= buf + len) + if (sbat == NULL || sbat < (char*)buf || sbat >= (char*)buf + len) return FALSE; for (i = 0; sbat[i] != '\0'; ) { diff --git a/src/parser.c b/src/parser.c index 2f000650464..94ece7998bb 100644 --- a/src/parser.c +++ b/src/parser.c @@ -1554,18 +1554,20 @@ sbat_entry_t* GetSbatEntries(char* sbatlevel) BOOL eol, eof; char* version_str; uint32_t i, num_entries; - sbat_entry_t* sbat_entries; + sbat_entry_t* _sbat_entries; if (sbatlevel == NULL) - return FALSE; + return NULL; num_entries = 0; for (i = 0; sbatlevel[i] != '\0'; i++) if (sbatlevel[i] == '\n') num_entries++; - sbat_entries = calloc(num_entries + 2, sizeof(sbat_entry_t)); - if (sbat_entries == NULL) + if (num_entries == 0) + return NULL; + _sbat_entries = calloc(num_entries + 2, sizeof(sbat_entry_t)); + if (_sbat_entries == NULL) return NULL; num_entries = 0; @@ -1581,7 +1583,7 @@ sbat_entry_t* GetSbatEntries(char* sbatlevel) i++; continue; } - sbat_entries[num_entries].product = &sbatlevel[i]; + _sbat_entries[num_entries].product = &sbatlevel[i]; for (; sbatlevel[i] != ',' && sbatlevel[i] != '\0' && sbatlevel[i] != '\n'; i++); if (sbatlevel[i] == '\0' || sbatlevel[i] == '\n') break; @@ -1595,22 +1597,35 @@ sbat_entry_t* GetSbatEntries(char* sbatlevel) i++; // Allow the provision of an hex version if (version_str[0] == '0' && version_str[1] == 'x') - sbat_entries[num_entries].version = strtoul(version_str, NULL, 16); + _sbat_entries[num_entries].version = strtoul(version_str, NULL, 16); else - sbat_entries[num_entries].version = strtoul(version_str, NULL, 10); + _sbat_entries[num_entries].version = strtoul(version_str, NULL, 10); if (!eol) for (; sbatlevel[i] != '\0' && sbatlevel[i] != '\n'; i++); - if (sbat_entries[num_entries].version != 0) + if (_sbat_entries[num_entries].version != 0) num_entries++; } - return sbat_entries; + return _sbat_entries; } /* * PE parsing functions */ +// Return the arch of a PE executable buffer +uint16_t GetPeArch(uint8_t* buf) +{ + IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; + IMAGE_NT_HEADERS* pe_header; + + if (buf == NULL) + return IMAGE_FILE_MACHINE_UNKNOWN; + + pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + return pe_header->FileHeader.Machine; +} + // Return the address and (optionally) the length of a PE section from a PE buffer uint8_t* GetPeSection(uint8_t* buf, const char* name, uint32_t* len) { diff --git a/src/resource.h b/src/resource.h index 90646d702b5..b8994a76aa2 100644 --- a/src/resource.h +++ b/src/resource.h @@ -69,6 +69,8 @@ #define IDR_LC_RUFUS_LOC 500 #define IDR_XT_HOGGER 501 #define IDR_UEFI_NTFS 502 +#define IDR_SETUP_X64 503 +#define IDR_SETUP_ARM64 504 // The following should match the ArchType array values + 600 #define IDR_MD5_BOOT 600 #define IDR_MD5_BOOTIA32 601 diff --git a/src/rufus.c b/src/rufus.c index dade331f510..4ba8c3785a4 100755 --- a/src/rufus.c +++ b/src/rufus.c @@ -1401,10 +1401,12 @@ static DWORD WINAPI BootCheckThread(LPVOID param) { int i, r, username_index = -1; FILE *fd; - DWORD len; + uint32_t len; + uint8_t* buf = NULL; WPARAM ret = BOOTCHECK_CANCEL; BOOL in_files_dir = FALSE, esp_already_asked = FALSE; BOOL is_windows_to_go = ((image_options & IMOP_WINTOGO) && (ComboBox_GetCurItemData(hImageOption) == IMOP_WIN_TO_GO)); + const char* msg; const char* grub = "grub"; const char* core_img = "core.img"; const char* ldlinux = "ldlinux"; @@ -1603,10 +1605,6 @@ static DWORD WINAPI BootCheckThread(LPVOID param) // Check UEFI bootloaders for revocation if (IS_EFI_BOOTABLE(img_report)) { - uint8_t* buf = NULL; - uint32_t len; - const char* msg; - for (i = 0; i < ARRAYSIZE(img_report.efi_boot_path) && img_report.efi_boot_path[i][0] != 0; i++) { static const char* revocation_type[] = { "UEFI DBX", "Windows SSP", "Linux SBAT", "Windows SVN" }; len = ReadISOFileToBuffer(image_path, img_report.efi_boot_path[i], &buf); @@ -1720,7 +1718,7 @@ static DWORD WINAPI BootCheckThread(LPVOID param) if ((partition_type == PARTITION_STYLE_MBR) && HAS_SYSLINUX(img_report)) { if (SL_MAJOR(img_report.sl_version) < 5) { IGNORE_RETVAL(_chdirU(app_data_dir)); - for (i = 0; i= 26000) { + setup_exe[0] = drive_letter; + setup_dll[0] = drive_letter; + dwSize = read_file(setup_exe, &buf); + if (dwSize != 0) { + setup_arch = GetPeArch(buf); + free(buf); + if (setup_arch != IMAGE_FILE_MACHINE_AMD64 && setup_arch != IMAGE_FILE_MACHINE_ARM64) { + uprintf("WARNING: Unsupported arch 0x%x -- in-place upgrade wrapper can not be added"); + } else if (!MoveFileExU(setup_exe, setup_dll, 0)) { + uprintf("Could not rename '%s': %s", setup_exe, WindowsErrorString()); + } else { + uprintf("Renamed '%s' → '%s'", setup_exe, setup_dll); + uprintf("Created '%s' bypass wrapper (from embedded)", setup_exe); + buf = GetResource(hMainInstance, MAKEINTRESOURCEA(setup_arch == IMAGE_FILE_MACHINE_AMD64 ? IDR_SETUP_X64 : IDR_SETUP_ARM64), + _RT_RCDATA, "setup.exe", &dwSize, FALSE); + if (buf == NULL) + uprintf("Could not access embedded 'setup.exe'"); + else + write_file(setup_exe, buf, dwSize); + } + } + } } UpdateProgressWithInfoForce(OP_PATCH, MSG_325, 0, PATCH_PROGRESS_TOTAL); From fd5c366938affd8e56cbda0b3369bcf42c152b06 Mon Sep 17 00:00:00 2001 From: Pete Batard Date: Wed, 9 Oct 2024 00:40:15 +0100 Subject: [PATCH 19/30] [wue] add experimental option to replace Windows bootloaders with the 24H2 _EX versions * This aims at creating installation media that is compatible with systems where 'Microsoft Windows Production PCA 2011' has been revoked. * Doesn't work, since the bootloaders being applied by the first stage installer come from \sources\install.wim[#]\windows\system32\Recovery\Winre.wim[#]\Windows\Boot\ (instead of \sources\boot.wim[#]\Windows\Boot\ as one would naturally expect) and Microsoft botched the ones they included there by using completely vulnerable (and therefore revoked) ones. See https://github.com/pbatard/rufus/issues/2244#issuecomment-2400380839. * Still, I sure haven't gone through this excruciating ACL bullshit for nothing, so you get an experimental option, behind the expert mode curtain. --- res/loc/rufus.loc | 1 + src/msapi_utf8.h | 73 +++++++++++++++++++++++ src/parser.c | 31 ++++++++-- src/rufus.c | 6 +- src/rufus.h | 12 +++- src/rufus.rc | 10 ++-- src/stdfn.c | 72 +++++++++++++++++++++++ src/stdio.c | 61 +++++++++++++++++++- src/wue.c | 143 ++++++++++++++++++++++++++++++++++------------ 9 files changed, 360 insertions(+), 49 deletions(-) diff --git a/res/loc/rufus.loc b/res/loc/rufus.loc index 0849e65b2d0..327443281b0 100644 --- a/res/loc/rufus.loc +++ b/res/loc/rufus.loc @@ -609,6 +609,7 @@ t MSG_346 "Restrict Windows to S-Mode (INCOMPATIBLE with online account bypass)" t MSG_347 "Expert Mode" t MSG_348 "Extracting archive files: %s" t MSG_349 "Use Rufus MBR" +t MSG_350 "Use 'Windows UEFI CA 2023' signed bootloaders [EXPERIMENTAL]" # The following messages are for the Windows Store listing only and are not used by the application t MSG_900 "Rufus is a utility that helps format and create bootable USB flash drives, such as USB keys/pendrives, memory sticks, etc." t MSG_901 "Official site: %s" diff --git a/src/msapi_utf8.h b/src/msapi_utf8.h index 7532c2624d4..e0510f410f4 100644 --- a/src/msapi_utf8.h +++ b/src/msapi_utf8.h @@ -26,6 +26,8 @@ #include #include #include +#include +#include #include #include #include @@ -89,6 +91,9 @@ static __inline char* wchar_to_utf8(const wchar_t* wstr) int size = 0; char* str = NULL; + if (wstr == NULL) + return NULL; + // Convert the empty string too if (wstr[0] == 0) return (char*)calloc(1, 1); @@ -568,6 +573,52 @@ static __inline const char* PathFindFileNameU(const char* szPath) return &szPath[i]; } +static __inline char* PathCombineU(char* lpDest, char* lpDir, char* lpFile) +{ + wchar_t* wret = NULL; + DWORD err = ERROR_INVALID_DATA; + wchar_t wlpDest[MAX_PATH]; + wconvert(lpDir); + wconvert(lpFile); + wret = PathCombineW(wlpDest, wlpDir, wlpFile); + err = GetLastError(); + wfree(lpDir); + wfree(lpFile); + if (wret == NULL) + return NULL; + wchar_to_utf8_no_alloc(wlpDest, lpDest, MAX_PATH); + SetLastError(err); + return lpDest; +} + +static __inline HANDLE FindFirstFileU(char* lpFileName, LPWIN32_FIND_DATAA lpFindFileData) +{ + HANDLE ret = INVALID_HANDLE_VALUE; + WIN32_FIND_DATAW wFindFileData = { 0 }; + wconvert(lpFileName); + ret = FindFirstFileW(wlpFileName, &wFindFileData); + if (ret != INVALID_HANDLE_VALUE) { + memcpy(lpFindFileData, &wFindFileData, offsetof(WIN32_FIND_DATAW, cFileName)); + wchar_to_utf8_no_alloc(wFindFileData.cFileName, lpFindFileData->cFileName, sizeof(lpFindFileData->cFileName)); + wchar_to_utf8_no_alloc(wFindFileData.cAlternateFileName, lpFindFileData->cAlternateFileName, sizeof(lpFindFileData->cAlternateFileName)); + } + wfree(lpFileName); + return ret; +} + +static __inline BOOL FindNextFileU(HANDLE hFindFile, LPWIN32_FIND_DATAA lpFindFileData) +{ + BOOL ret = FALSE; + WIN32_FIND_DATAW wFindFileData = { 0 }; + ret = FindNextFileW(hFindFile, &wFindFileData); + if (ret) { + memcpy(lpFindFileData, &wFindFileData, offsetof(WIN32_FIND_DATAW, cFileName)); + wchar_to_utf8_no_alloc(wFindFileData.cFileName, lpFindFileData->cFileName, sizeof(lpFindFileData->cFileName)); + wchar_to_utf8_no_alloc(wFindFileData.cAlternateFileName, lpFindFileData->cAlternateFileName, sizeof(lpFindFileData->cAlternateFileName)); + } + return ret; +} + // This function differs from regular GetTextExtentPoint in that it uses a zero terminated string static __inline BOOL GetTextExtentPointU(HDC hdc, const char* lpString, LPSIZE lpSize) { @@ -819,6 +870,28 @@ static __inline BOOL SetFileAttributesU(const char* lpFileName, DWORD dwFileAttr return ret; } +static __inline DWORD GetNamedSecurityInfoU(const char* lpObjectName, SE_OBJECT_TYPE ObjectType, + SECURITY_INFORMATION SecurityInfo, PSID* ppsidOwner, PSID* ppsidGroup, PACL* ppDacl, + PACL* ppSacl, PSECURITY_DESCRIPTOR* ppSecurityDescriptor) +{ + DWORD ret; + wconvert(lpObjectName); + ret = GetNamedSecurityInfoW(wlpObjectName, ObjectType, SecurityInfo, ppsidOwner, ppsidGroup, + ppDacl, ppSacl, ppSecurityDescriptor); + wfree(lpObjectName); + return ret; +} + +static __inline DWORD SetNamedSecurityInfoU(const char* lpObjectName, SE_OBJECT_TYPE ObjectType, + SECURITY_INFORMATION SecurityInfo, PSID psidOwner, PSID psidGroup, PACL pDacl, PACL pSacl) +{ + DWORD ret; + wconvert(lpObjectName); + ret = SetNamedSecurityInfoW(wlpObjectName, ObjectType, SecurityInfo, psidOwner, psidGroup, pDacl, pSacl); + wfree(lpObjectName); + return ret; +} + static __inline int SHCreateDirectoryExU(HWND hwnd, const char* pszPath, SECURITY_ATTRIBUTES *psa) { int ret = ERROR_INVALID_DATA; diff --git a/src/parser.c b/src/parser.c index 94ece7998bb..ca48c61e4cf 100644 --- a/src/parser.c +++ b/src/parser.c @@ -1267,7 +1267,7 @@ char* replace_in_token_data(const char* filename, const char* token, const char* } /* - * Replace all 'c' characters in string 'src' with the substring 'rep' + * Replace all 'c' characters in string 'src' with the substring 'rep'. * The returned string is allocated and must be freed by the caller. */ char* replace_char(const char* src, const char c, const char* rep) @@ -1300,6 +1300,30 @@ char* replace_char(const char* src, const char c, const char* rep) return res; } +/* + * Remove all instances of substring 'sub' form string 'src. + * The returned string is allocated and must be freed by the caller. + */ +char* remove_substr(const char* src, const char* sub) +{ + size_t i, j, str_len = safe_strlen(src), sub_len = safe_strlen(sub); + char* res; + + if ((src == NULL) || (sub == NULL) || (sub_len > str_len)) + return NULL; + + res = (char*)calloc(str_len + 1, 1); + if (res == NULL) + return NULL; + for (i = 0, j = 0; i <= str_len; ) { + if (i <= str_len - sub_len && memcmp(&src[i], sub, sub_len) == 0) + i += sub_len; + else + res[j++] = src[i++]; + } + return res; +} + /* * Internal recursive call for get_data_from_asn1(). Returns FALSE on error, TRUE otherwise. */ @@ -1697,8 +1721,7 @@ uint8_t* RvaToPhysical(uint8_t* buf, uint32_t rva) static BOOL FoundResourceRva = FALSE; uint32_t FindResourceRva(const wchar_t* name, uint8_t* root, uint8_t* dir, uint32_t* len) { - uint32_t rva; - WORD i; + uint32_t i, rva; IMAGE_RESOURCE_DIRECTORY* _dir = (IMAGE_RESOURCE_DIRECTORY*)dir; IMAGE_RESOURCE_DIRECTORY_ENTRY* dir_entry = (IMAGE_RESOURCE_DIRECTORY_ENTRY*)&_dir[1]; IMAGE_RESOURCE_DIR_STRING_U* dir_string; @@ -1711,7 +1734,7 @@ uint32_t FindResourceRva(const wchar_t* name, uint8_t* root, uint8_t* dir, uint3 if (root == dir) FoundResourceRva = FALSE; - for (i = 0; i < _dir->NumberOfNamedEntries + _dir->NumberOfIdEntries; i++) { + for (i = 0; i < (uint32_t)_dir->NumberOfNamedEntries + _dir->NumberOfIdEntries; i++) { if (!FoundResourceRva && i < _dir->NumberOfNamedEntries) { dir_string = (IMAGE_RESOURCE_DIR_STRING_U*)(root + dir_entry[i].NameOffset); if (dir_string->Length != wcslen(name) || diff --git a/src/rufus.c b/src/rufus.c index 4ba8c3785a4..8e0837342a6 100755 --- a/src/rufus.c +++ b/src/rufus.c @@ -1586,6 +1586,10 @@ static DWORD WINAPI BootCheckThread(LPVOID param) StrArrayAdd(&options, lmprintf(MSG_335), TRUE); MAP_BIT(UNATTEND_DISABLE_BITLOCKER); if (expert_mode) { + if (!appstore_version && img_report.win_version.build >= 26100) { + StrArrayAdd(&options, lmprintf(MSG_350), TRUE); + MAP_BIT(UNATTEND_USE_MS2023_BOOTLOADERS); + } StrArrayAdd(&options, lmprintf(MSG_346), TRUE); MAP_BIT(UNATTEND_FORCE_S_MODE); } @@ -1597,7 +1601,7 @@ static DWORD WINAPI BootCheckThread(LPVOID param) i = remap16(i, map, TRUE); unattend_xml_path = CreateUnattendXml(arch, i); // Remember the user preferences for the current session. - unattend_xml_mask &= ~(remap16(0x1ff, map, TRUE)); + unattend_xml_mask &= ~(remap16(UNATTEND_FULL_MASK, map, TRUE)); unattend_xml_mask |= i; WriteSetting32(SETTING_WUE_OPTIONS, (UNATTEND_DEFAULT_MASK << 16) | unattend_xml_mask); } diff --git a/src/rufus.h b/src/rufus.h index 24a4dfb023f..9240784a23e 100644 --- a/src/rufus.h +++ b/src/rufus.h @@ -629,6 +629,8 @@ typedef struct { #define UNATTEND_SET_USER 0x00040 #define UNATTEND_DISABLE_BITLOCKER 0x00080 #define UNATTEND_FORCE_S_MODE 0x00100 +#define UNATTEND_USE_MS2023_BOOTLOADERS 0x00200 +#define UNATTEND_FULL_MASK 0x003FF #define UNATTEND_DEFAULT_MASK 0x000FF #define UNATTEND_WINDOWS_TO_GO 0x10000 // Special flag for Windows To Go @@ -636,10 +638,15 @@ typedef struct { #define UNATTEND_SPECIALIZE_DEPLOYMENT_MASK (UNATTEND_NO_ONLINE_ACCOUNT) #define UNATTEND_OOBE_SHELL_SETUP_MASK (UNATTEND_NO_DATA_COLLECTION | UNATTEND_SET_USER | UNATTEND_DUPLICATE_LOCALE) #define UNATTEND_OOBE_INTERNATIONAL_MASK (UNATTEND_DUPLICATE_LOCALE) -#define UNATTEND_OOBE_MASK (UNATTEND_OOBE_SHELL_SETUP_MASK | UNATTEND_OOBE_INTERNATIONAL_MASK | UNATTEND_DISABLE_BITLOCKER) +#define UNATTEND_OOBE_MASK (UNATTEND_OOBE_SHELL_SETUP_MASK | UNATTEND_OOBE_INTERNATIONAL_MASK | UNATTEND_DISABLE_BITLOCKER | UNATTEND_USE_MS2023_BOOTLOADERS) #define UNATTEND_OFFLINE_SERVICING_MASK (UNATTEND_OFFLINE_INTERNAL_DRIVES | UNATTEND_FORCE_S_MODE) #define UNATTEND_DEFAULT_SELECTION_MASK (UNATTEND_SECUREBOOT_TPM_MINRAM | UNATTEND_NO_ONLINE_ACCOUNT | UNATTEND_OFFLINE_INTERNAL_DRIVES) +/* Used with ListDirectoryContent */ +#define LIST_DIR_TYPE_FILE 0x01 +#define LIST_DIR_TYPE_DIRECTORY 0x02 +#define LIST_DIR_TYPE_RECURSIVE 0x80 + /* Hash tables */ typedef struct htab_entry { uint32_t used; @@ -786,6 +793,7 @@ extern char* get_token_data_buffer(const char* token, unsigned int n, const char extern char* insert_section_data(const char* filename, const char* section, const char* data, BOOL dos2unix); extern char* replace_in_token_data(const char* filename, const char* token, const char* src, const char* rep, BOOL dos2unix); extern char* replace_char(const char* src, const char c, const char* rep); +extern char* remove_substr(const char* src, const char* sub); extern void parse_update(char* buf, size_t len); extern void* get_data_from_asn1(const uint8_t* buf, size_t buf_len, const char* oid_str, uint8_t asn1_type, size_t* data_len); extern int sanitize_label(char* label); @@ -835,6 +843,8 @@ extern uint16_t GetPeArch(uint8_t* buf); extern uint8_t* GetPeSection(uint8_t* buf, const char* name, uint32_t* len); extern uint8_t* RvaToPhysical(uint8_t* buf, uint32_t rva); extern uint32_t FindResourceRva(const wchar_t* name, uint8_t* root, uint8_t* dir, uint32_t* len); +extern DWORD ListDirectoryContent(StrArray* arr, char* dir, uint8_t type); +extern BOOL TakeOwnership(LPCSTR lpszOwnFile); #define GetTextWidth(hDlg, id) GetTextSize(GetDlgItem(hDlg, id), NULL).cx DWORD WINAPI HashThread(void* param); diff --git a/src/rufus.rc b/src/rufus.rc index f63f929483c..7236b2c2e6c 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2200" +CAPTION "Rufus 4.6.2201" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -399,8 +399,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2200,0 - PRODUCTVERSION 4,6,2200,0 + FILEVERSION 4,6,2201,0 + PRODUCTVERSION 4,6,2201,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -418,13 +418,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2200" + VALUE "FileVersion", "4.6.2201" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", " 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2200" + VALUE "ProductVersion", "4.6.2201" END END BLOCK "VarFileInfo" diff --git a/src/stdfn.c b/src/stdfn.c index ec4933a03ca..c299967b8ee 100644 --- a/src/stdfn.c +++ b/src/stdfn.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include "re.h" #include "rufus.h" @@ -1242,3 +1244,73 @@ BOOL UnmountRegistryHive(const HKEY key, const char* pszHiveName) return (status == ERROR_SUCCESS); } + +/* + * Take administrative ownership of a file or directory, and grant all access rights. + */ +BOOL TakeOwnership(LPCSTR lpszOwnFile) +{ + BOOL ret = FALSE; + HANDLE hToken = NULL; + PSID pSIDAdmin = NULL; + PACL pOldDACL = NULL, pNewDACL = NULL; + PSECURITY_DESCRIPTOR pSD = NULL; + SID_IDENTIFIER_AUTHORITY SIDAuthNT = SECURITY_NT_AUTHORITY; + EXPLICIT_ACCESS ea = { 0 }; + + if (lpszOwnFile == NULL) + return FALSE; + + // Create a SID for the BUILTIN\Administrators group. + if (!AllocateAndInitializeSid(&SIDAuthNT, 2, SECURITY_BUILTIN_DOMAIN_RID, + DOMAIN_ALIAS_RID_ADMINS, 0, 0, 0, 0, 0, 0, &pSIDAdmin)) + goto out; + + // Open a handle to the access token for the calling process. + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &hToken)) + goto out; + + // Enable the SE_TAKE_OWNERSHIP_NAME privilege. + if (!SetPrivilege(hToken, SE_TAKE_OWNERSHIP_NAME, TRUE)) + goto out; + + // Set the owner in the object's security descriptor. + if (SetNamedSecurityInfoU(lpszOwnFile, SE_FILE_OBJECT, OWNER_SECURITY_INFORMATION, + pSIDAdmin, NULL, NULL, NULL) != ERROR_SUCCESS) + goto out; + + // Disable the SE_TAKE_OWNERSHIP_NAME privilege. + if (!SetPrivilege(hToken, SE_TAKE_OWNERSHIP_NAME, FALSE)) + goto out; + + // Get a pointer to the existing DACL. + if (GetNamedSecurityInfoU(lpszOwnFile, SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, + NULL, NULL, &pOldDACL, NULL, &pSD) != ERROR_SUCCESS) + goto out; + + // Initialize an EXPLICIT_ACCESS structure for the new ACE + // with full control for Administrators. + ea.grfAccessPermissions = GENERIC_ALL; + ea.grfAccessMode = GRANT_ACCESS; + ea.grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; + ea.Trustee.TrusteeForm = TRUSTEE_IS_SID; + ea.Trustee.TrusteeType = TRUSTEE_IS_GROUP; + ea.Trustee.ptstrName = (LPTSTR)pSIDAdmin; + + // Create a new ACL that merges the new ACE into the existing DACL. + if (SetEntriesInAcl(1, &ea, pOldDACL, &pNewDACL) != ERROR_SUCCESS) + goto out; + + // Try to modify the object's DACL. + if (SetNamedSecurityInfoU(lpszOwnFile, SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, + NULL, NULL, pNewDACL, NULL) != ERROR_SUCCESS) + goto out; + + ret = TRUE; + +out: + FreeSid(pSIDAdmin); + LocalFree(pNewDACL); + safe_closehandle(hToken); + return ret; +} diff --git a/src/stdio.c b/src/stdio.c index 4004575189f..a6c257b403e 100644 --- a/src/stdio.c +++ b/src/stdio.c @@ -203,7 +203,7 @@ void DumpBufferHex(void *buf, size_t size) uprintf("%s\n", line); line[0] = 0; sprintf(&line[strlen(line)], " %08x ", (unsigned int)i); - for(j=0,k=0; k<16; j++,k++) { + for(j = 0,k = 0; k < 16; j++,k++) { if (i+j < size) { sprintf(&line[strlen(line)], "%02x", buffer[i+j]); } else { @@ -212,7 +212,7 @@ void DumpBufferHex(void *buf, size_t size) sprintf(&line[strlen(line)], " "); } sprintf(&line[strlen(line)], " "); - for(j=0,k=0; k<16; j++,k++) { + for(j = 0,k = 0; k < 16; j++,k++) { if (i+j < size) { if ((buffer[i+j] < 32) || (buffer[i+j] > 126)) { sprintf(&line[strlen(line)], "."); @@ -920,3 +920,60 @@ BOOL ExtractZip(const char* src_zip, const char* dest_dir) bled_exit(); return (extracted_bytes > 0); } + +// Returns a list of all the files or folders from a directory +DWORD ListDirectoryContent(StrArray* arr, char* dir, uint8_t type) +{ + WIN32_FIND_DATAA FindFileData = { 0 }; + HANDLE hFind; + DWORD dwError, dwResult; + char mask[MAX_PATH + 1], path[MAX_PATH + 1]; + + if (arr == NULL || dir == NULL || (type & 0x03) == 0) + return ERROR_INVALID_PARAMETER; + + if (PathCombineU(mask, dir, "*") == NULL) + return GetLastError(); + + hFind = FindFirstFileU(mask, &FindFileData); + if (hFind == INVALID_HANDLE_VALUE) + return GetLastError(); + + dwResult = ERROR_FILE_NOT_FOUND; + do { + if (FindFileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + if (strcmp(FindFileData.cFileName, ".") == 0 || + strcmp(FindFileData.cFileName, "..") == 0 || + (type & LIST_DIR_TYPE_RECURSIVE) == 0) + continue; + if (PathCombineU(path, dir, FindFileData.cFileName) == NULL) + break; + // Append a trailing backslash to directories + if (path[strlen(path) - 1] != '\\') { + path[strlen(path) + 1] = '\0'; + path[strlen(path)] = '\\'; + } + if (type & LIST_DIR_TYPE_DIRECTORY) + StrArrayAdd(arr, path, TRUE); + dwError = ListDirectoryContent(arr, path, type); + if (dwError != NO_ERROR && dwError != ERROR_FILE_NOT_FOUND) { + SetLastError(dwError); + break; + } + } else { + if (type & LIST_DIR_TYPE_FILE) { + if (PathCombineU(path, dir, FindFileData.cFileName) == NULL) + break; + StrArrayAdd(arr, path, TRUE); + } + dwResult = NO_ERROR; + } + } while (FindNextFileU(hFind, &FindFileData)); + + dwError = GetLastError(); + FindClose(hFind); + + if (dwError != ERROR_NO_MORE_FILES) + return dwError; + return dwResult; +} diff --git a/src/wue.c b/src/wue.c index 6318aedee7d..46db5bfd5be 100644 --- a/src/wue.c +++ b/src/wue.c @@ -51,6 +51,7 @@ extern uint32_t wim_nb_files, wim_proc_files, wim_extra_files; extern BOOL validate_md5sum; extern uint64_t md5sum_totalbytes; extern StrArray modified_files; +extern const char* efi_archname[ARCH_MAX]; /// /// Create an installation answer file containing the sections specified by the flags. @@ -162,43 +163,56 @@ char* CreateUnattendXml(int arch, int flags) free(tzstr); } } - if (flags & UNATTEND_SET_USER) { - for (i = 0; (i < ARRAYSIZE(unallowed_account_names)) && (stricmp(unattend_username, unallowed_account_names[i]) != 0); i++); - if (i < ARRAYSIZE(unallowed_account_names)) { - uprintf("WARNING: '%s' is not allowed as local account name - Option ignored", unattend_username); - } else if (unattend_username[0] != 0) { - uprintf("• Use '%s' for local account name", unattend_username); - // If we create a local account in unattend.xml, then we can get Windows 11 - // 22H2 to skip MSA even if the network is connected during installation. - fprintf(fd, " \n"); - fprintf(fd, " \n"); - fprintf(fd, " \n"); - fprintf(fd, " %s\n", unattend_username); - fprintf(fd, " %s\n", unattend_username); - fprintf(fd, " Administrators;Power Users\n"); - // Sets an empty password for the account (which, in Microsoft's convoluted ways, - // needs to be initialized to the Base64 encoded UTF-16 string "Password"). - // The use of an empty password has both the advantage of not having to ask users - // to type in a password in Rufus (which they might be weary of) as well as allowing - // automated logon during setup. - fprintf(fd, " \n"); - fprintf(fd, " UABhAHMAcwB3AG8AcgBkAA==\n"); - fprintf(fd, " false</PlainText>\n"); - fprintf(fd, " </Password>\n"); - fprintf(fd, " </LocalAccount>\n"); - fprintf(fd, " </LocalAccounts>\n"); - fprintf(fd, " </UserAccounts>\n"); - // Since we set a blank password, we'll ask the user to change it at next logon. + if (flags & UNATTEND_SET_USER || flags & UNATTEND_USE_MS2023_BOOTLOADERS) { + if (flags & UNATTEND_SET_USER) { + for (i = 0; (i < ARRAYSIZE(unallowed_account_names)) && (stricmp(unattend_username, unallowed_account_names[i]) != 0); i++); + if (i < ARRAYSIZE(unallowed_account_names)) { + uprintf("WARNING: '%s' is not allowed as local account name - Option ignored", unattend_username); + } else if (unattend_username[0] != 0) { + uprintf("• Use '%s' for local account name", unattend_username); + // If we create a local account in unattend.xml, then we can get Windows 11 + // 22H2 to skip MSA even if the network is connected during installation. + fprintf(fd, " <UserAccounts>\n"); + fprintf(fd, " <LocalAccounts>\n"); + fprintf(fd, " <LocalAccount wcm:action=\"add\">\n"); + fprintf(fd, " <Name>%s</Name>\n", unattend_username); + fprintf(fd, " <DisplayName>%s</DisplayName>\n", unattend_username); + fprintf(fd, " <Group>Administrators;Power Users</Group>\n"); + // Sets an empty password for the account (which, in Microsoft's convoluted ways, + // needs to be initialized to the Base64 encoded UTF-16 string "Password"). + // The use of an empty password has both the advantage of not having to ask users + // to type in a password in Rufus (which they might be weary of) as well as allowing + // automated logon during setup. + fprintf(fd, " <Password>\n"); + fprintf(fd, " <Value>UABhAHMAcwB3AG8AcgBkAA==</Value>\n"); + fprintf(fd, " <PlainText>false</PlainText>\n"); + fprintf(fd, " </Password>\n"); + fprintf(fd, " </LocalAccount>\n"); + fprintf(fd, " </LocalAccounts>\n"); + fprintf(fd, " </UserAccounts>\n"); + // Since we set a blank password, we'll ask the user to change it at next logon. + fprintf(fd, " <FirstLogonCommands>\n"); + fprintf(fd, " <SynchronousCommand wcm:action=\"add\">\n"); + fprintf(fd, " <Order>%d</Order>\n", order++); + fprintf(fd, " <CommandLine>net user &quot;%s&quot; /logonpasswordchg:yes</CommandLine>\n", unattend_username); + fprintf(fd, " </SynchronousCommand>\n"); + // Some people report that using the `net user` command above might reset the password expiration to 90 days... + // To alleviate that, blanket set passwords on the target machine to never expire. + fprintf(fd, " <SynchronousCommand wcm:action=\"add\">\n"); + fprintf(fd, " <Order>%d</Order>\n", order++); + fprintf(fd, " <CommandLine>net accounts /maxpwage:unlimited</CommandLine>\n"); + fprintf(fd, " </SynchronousCommand>\n"); + fprintf(fd, " </FirstLogonCommands>\n"); + } + } + if (flags & UNATTEND_USE_MS2023_BOOTLOADERS) { + uprintf("• Use 'Windows UEFI CA 2023' signed bootloaders"); + // TODO: Validate that we can have multiple <FirstLogonCommands> sections fprintf(fd, " <FirstLogonCommands>\n"); fprintf(fd, " <SynchronousCommand wcm:action=\"add\">\n"); fprintf(fd, " <Order>%d</Order>\n", order++); - fprintf(fd, " <CommandLine>net user &quot;%s&quot; /logonpasswordchg:yes</CommandLine>\n", unattend_username); - fprintf(fd, " </SynchronousCommand>\n"); - // Some people report that using the `net user` command above might reset the password expiration to 90 days... - // To alleviate that, blanket set passwords on the target machine to never expire. - fprintf(fd, " <SynchronousCommand wcm:action=\"add\">\n"); - fprintf(fd, " <Order>%d</Order>\n", order++); - fprintf(fd, " <CommandLine>net accounts /maxpwage:unlimited</CommandLine>\n"); + // TODO: Validate the actual value on a machine where updates have been applied + fprintf(fd, " <CommandLine>reg add HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Secureboot /v AvailableUpdates /t REG_DWORD /d 0x3c0 /f\n"); fprintf(fd, " </SynchronousCommand>\n"); fprintf(fd, " </FirstLogonCommands>\n"); } @@ -845,7 +859,7 @@ BOOL ApplyWindowsCustomization(char drive_letter, int flags) setup_arch = GetPeArch(buf); free(buf); if (setup_arch != IMAGE_FILE_MACHINE_AMD64 && setup_arch != IMAGE_FILE_MACHINE_ARM64) { - uprintf("WARNING: Unsupported arch 0x%x -- in-place upgrade wrapper can not be added"); + uprintf("WARNING: Unsupported arch 0x%x -- in-place upgrade wrapper will not be added", setup_arch); } else if (!MoveFileExU(setup_exe, setup_dll, 0)) { uprintf("Could not rename '%s': %s", setup_exe, WindowsErrorString()); } else { @@ -865,7 +879,8 @@ BOOL ApplyWindowsCustomization(char drive_letter, int flags) UpdateProgressWithInfoForce(OP_PATCH, MSG_325, 0, PATCH_PROGRESS_TOTAL); // We only need to mount boot.wim if we have windowsPE data to deal with. If // not, we can just copy our unattend.xml in \sources\$OEM$\$$\Panther\. - if (flags & UNATTEND_WINPE_SETUP_MASK) { + // We also need to mount it if we use the 'Windows UEFI CA 2023' signed bootloaders. + if (flags & UNATTEND_WINPE_SETUP_MASK || flags & UNATTEND_USE_MS2023_BOOTLOADERS) { if (validate_md5sum) md5sum_totalbytes -= _filesizeU(boot_wim_path); uprintf("Mounting '%s[%d]'...", boot_wim_path, wim_index); @@ -969,6 +984,62 @@ BOOL ApplyWindowsCustomization(char drive_letter, int flags) } UpdateProgressWithInfoForce(OP_PATCH, MSG_325, 103, PATCH_PROGRESS_TOTAL); } + + if (flags & UNATTEND_USE_MS2023_BOOTLOADERS) { + if_not_assert(mount_path != NULL) + goto out; + static_sprintf(path, "%s\\Windows\\Boot\\EFI_EX\\bootmgfw_EX.efi", mount_path); + if (!PathFileExistsU(path)) { + uprintf("Could not find 2023 signed UEFI bootloader - Ignoring option"); + } else { + char path2[MAX_PATH], *rep; + StrArray files, dirs; + // Replace /EFI/Boot/boot###.efi + for (i = 1; i < ARRAYSIZE(efi_archname); i++) { + static_sprintf(path2, "%c:\\efi\\boot\\boot%s.efi", drive_letter, efi_archname[i]); + if (!PathFileExistsA(path2)) + continue; + if (!CopyFileU(path, path2, FALSE)) + uprintf("WARNING: Could not replace 'boot%s.efi': %s", efi_archname[i], WindowsErrorString()); + break; + } + // Replace /bootmgr.efi + static_sprintf(path, "%s\\Windows\\Boot\\EFI_EX\\bootmgr_EX.efi", mount_path); + static_sprintf(path2, "%c:\\bootmgr.efi", drive_letter); + if (!CopyFileU(path, path2, FALSE)) + uprintf("WARNING: Could not replace 'bootmgr.efi': %s", WindowsErrorString()); + // Microsoft "secures" the Windows\Boot\ dir through their SUPER OBNOXIOUS AND + // WORTHLESS use of DACLs + read-only flags, so we first need to re-take control + // of all directories under there recursively. + StrArrayCreate(&dirs, 64); + StrArrayCreate(&files, 64); + static_sprintf(path, "%s\\Windows\\Boot\\", mount_path); + StrArrayAdd(&dirs, path, TRUE); + static_sprintf(path, "%s\\Windows\\Boot\\EFI_EX\\", mount_path); + StrArrayAdd(&dirs, path, TRUE); + ListDirectoryContent(&dirs, path, LIST_DIR_TYPE_DIRECTORY | LIST_DIR_TYPE_RECURSIVE); + for (i = 0; i < (int)dirs.Index; i++) { + rep = remove_substr(dirs.String[i], "_EX"); + assert(rep != NULL); + TakeOwnership(rep); + safe_free(rep); + } + // Now that we should be able to write to the destination directories, copy the content. + ListDirectoryContent(&files, path, LIST_DIR_TYPE_FILE | LIST_DIR_TYPE_RECURSIVE); + for (i = 0; r && i < (int)files.Index; i++) { + rep = remove_substr(files.String[i], "_EX"); + assert(rep != NULL); + TakeOwnership(rep); + if (!CopyFileU(files.String[i], rep, FALSE) && rep != NULL) + uprintf("WARNING: Could not replace '%s': %s", &rep[strlen(mount_path) + 1], WindowsErrorString()); + safe_free(rep); + } + StrArrayDestroy(&dirs); + StrArrayDestroy(&files); + uprintf("Replaced EFI bootloader files with 'Windows UEFI CA 2023' signed versions"); + } + } + r = TRUE; out: From 6b5837dbb5fc97b72770ccd792bb38ba5c583b7e Mon Sep 17 00:00:00 2001 From: Pete Batard <pete@akeo.ie> Date: Wed, 9 Oct 2024 13:20:30 +0100 Subject: [PATCH 20/30] [iso] add DBX certificate revocation validation and reporting * This is currently only used to detect the use of 'Microsoft Windows Production PCA 2011' signed bootloaders. * Because the cert is still in the process of being revoked, and Windows 11 24H2 still uses 'Microsoft Windows Production PCA 2011' signed bootloaders by default, only report this if running in expert mode. * Also fix non-reachable code in wue.c. --- src/db.h | 10 ++++- src/hash.c | 37 ++++++++++++++-- src/parser.c | 39 +++++++++++++++-- src/pki.c | 121 ++++++++++++++++++++++++++++++++++++++++++++++----- src/rufus.c | 2 +- src/rufus.h | 10 ++++- src/rufus.rc | 10 ++--- src/wue.c | 2 +- 8 files changed, 204 insertions(+), 27 deletions(-) diff --git a/src/db.h b/src/db.h index 30aa301ae4d..52c026598e2 100644 --- a/src/db.h +++ b/src/db.h @@ -2,7 +2,7 @@ * Rufus: The Reliable USB Formatting Utility * DB of the known SHA256 hash values for Rufus downloadable content (GRUB, Syslinux, etc.) * as well PE256 hash values for UEFI revoked content (DBX, SkuSiPolicy.p7b) - * Copyright © 2016-2023 Pete Batard <pete@akeo.ie> + * Copyright © 2016-2024 Pete Batard <pete@akeo.ie> * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -657,6 +657,14 @@ static uint8_t pe256dbx[] = { 0xff, 0xf4, 0x21, 0xa9, 0xdc, 0xd3, 0xef, 0x38, 0xad, 0x58, 0x5e, 0x8b, 0xac, 0xa4, 0x08, 0xac, 0x2e, 0x4c, 0xdb, 0xdf, 0xa6, 0x79, 0x90, 0x0e, 0xc1, 0x70, 0x89, 0x62, 0x4e, 0x31, 0x0a, 0xda, }; +/* + * Contains the SHA-1 thumbprint of certificates that are being revoked by DBX. + * This only includes the 'Microsoft Windows Production PCA 2011' for now. + */ +static uint8_t certdbx[] = { + 0x58, 0x0a, 0x6f, 0x4c, 0xc4, 0xe4, 0xb6, 0x69, 0xb9, 0xeb, 0xdc, 0x1b, 0x2b, 0x3e, 0x08, 0x7b, 0x80, 0xd0, 0x67, 0x8d +}; + /* * Extended SBATLevel.txt that merges Linux SBAT with Microsoft's SVN * See https://github.com/pbatard/rufus/issues/2244#issuecomment-2243661539 diff --git a/src/hash.c b/src/hash.c index d39f68abaad..5ff7df1061b 100644 --- a/src/hash.c +++ b/src/hash.c @@ -117,6 +117,7 @@ StrArray modified_files = { 0 }; extern int default_thread_priority; extern const char* efi_archname[ARCH_MAX]; extern char* sbat_level_txt; +extern BOOL expert_mode; /* * Rotate 32 or 64 bit integers by n bytes. @@ -2166,10 +2167,35 @@ BOOL IsRevokedBySvn(uint8_t* buf, uint32_t len) return FALSE; } +BOOL IsRevokedByCert(uint8_t* buf, uint32_t len) +{ + int i; + uint8_t* cert; + cert_info_t info; + + cert = GetPeSignatureData(buf); + if (cert == NULL) + return FALSE; + if (!GetIssuerCertificateInfo(cert, &info)) + return FALSE; + + for (i = 0; i < ARRAYSIZE(certdbx); i += SHA1_HASHSIZE) { + if (!expert_mode) + continue; + if (memcmp(info.thumbprint, &certdbx[i], SHA1_HASHSIZE) == 0) { + uprintf("Found '%s' revoked certificate", info.name); + return TRUE; + } + } + return FALSE; +} + int IsBootloaderRevoked(uint8_t* buf, uint32_t len) { uint32_t i; uint8_t hash[SHA256_HASHSIZE]; + IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; + IMAGE_NT_HEADERS* pe_header; // Fall back to embedded sbat_level.txt if we couldn't access remote if (sbat_entries == NULL) { @@ -2177,8 +2203,10 @@ int IsBootloaderRevoked(uint8_t* buf, uint32_t len) sbat_entries = GetSbatEntries(sbat_level_txt); } - // TODO: More elaborate PE checks? - if (buf == NULL || len < 0x100 || buf[0] != 'M' || buf[1] != 'Z') + if (buf == NULL || len < 0x100 || dos_header->e_magic != IMAGE_DOS_SIGNATURE) + return -2; + pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + if (pe_header->Signature != IMAGE_NT_SIGNATURE) return -2; if (!PE256Buffer(buf, len, hash)) return -1; @@ -2193,9 +2221,12 @@ int IsBootloaderRevoked(uint8_t* buf, uint32_t len) // Check for Linux SBAT revocation if (IsRevokedBySbat(buf, len)) return 3; - // Sheck for Microsoft SVN revocation + // Check for Microsoft SVN revocation if (IsRevokedBySvn(buf, len)) return 4; + // Check for DBX certificate revocation + if (IsRevokedByCert(buf, len)) + return 5; return 0; } diff --git a/src/parser.c b/src/parser.c index ca48c61e4cf..7bc21692ddf 100644 --- a/src/parser.c +++ b/src/parser.c @@ -24,6 +24,8 @@ #endif #include <windows.h> +#include <wincrypt.h> +#include <wintrust.h> #include <stdio.h> #include <wchar.h> #include <string.h> @@ -1643,10 +1645,12 @@ uint16_t GetPeArch(uint8_t* buf) IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; IMAGE_NT_HEADERS* pe_header; - if (buf == NULL) + if (buf == NULL || dos_header->e_magic != IMAGE_DOS_SIGNATURE) return IMAGE_FILE_MACHINE_UNKNOWN; pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + if (pe_header->Signature != IMAGE_NT_SIGNATURE) + return IMAGE_FILE_MACHINE_UNKNOWN; return pe_header->FileHeader.Machine; } @@ -1662,10 +1666,12 @@ uint8_t* GetPeSection(uint8_t* buf, const char* name, uint32_t* len) static_strcpy(section_name, name); - if (buf == NULL || name == NULL) + if (buf == NULL || name == NULL || dos_header->e_magic != IMAGE_DOS_SIGNATURE) return NULL; pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + if (pe_header->Signature != IMAGE_NT_SIGNATURE) + return NULL; if (pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_I386 || pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_ARM) { section_header = (IMAGE_SECTION_HEADER*)(&pe_header[1]); nb_sections = pe_header->FileHeader.NumberOfSections; @@ -1693,10 +1699,12 @@ uint8_t* RvaToPhysical(uint8_t* buf, uint32_t rva) IMAGE_NT_HEADERS64* pe64_header; IMAGE_SECTION_HEADER* section_header; - if (buf == NULL) + if (buf == NULL || dos_header->e_magic != IMAGE_DOS_SIGNATURE) return NULL; pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + if (pe_header->Signature != IMAGE_NT_SIGNATURE) + return NULL; if (pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_I386 || pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_ARM) { section_header = (IMAGE_SECTION_HEADER*)(pe_header + 1); nb_sections = pe_header->FileHeader.NumberOfSections; @@ -1755,3 +1763,28 @@ uint32_t FindResourceRva(const wchar_t* name, uint8_t* root, uint8_t* dir, uint3 } return 0; } + +uint8_t* GetPeSignatureData(uint8_t* buf) +{ + IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; + IMAGE_NT_HEADERS* pe_header; + IMAGE_DATA_DIRECTORY sec_dir; + WIN_CERTIFICATE* cert; + + if (buf == NULL || dos_header->e_magic != IMAGE_DOS_SIGNATURE) + return NULL; + + pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + if (pe_header->Signature != IMAGE_NT_SIGNATURE) + return NULL; + + sec_dir = pe_header->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_SECURITY]; + if (sec_dir.VirtualAddress == 0 || sec_dir.Size == 0) + return NULL; + + cert = (WIN_CERTIFICATE*)&buf[sec_dir.VirtualAddress]; + if (cert->dwLength == 0 || cert->wCertificateType != WIN_CERT_TYPE_PKCS_SIGNED_DATA) + return NULL; + + return (uint8_t*)cert; +} diff --git a/src/pki.c b/src/pki.c index 5c69d3fce8b..09dda43c80f 100644 --- a/src/pki.c +++ b/src/pki.c @@ -271,9 +271,9 @@ char* GetSignatureName(const char* path, const char* country_code, BOOL bSilent) HCERTSTORE hStore = NULL; HCRYPTMSG hMsg = NULL; PCCERT_CONTEXT pCertContext = NULL; - DWORD dwSize, dwEncoding, dwContentType, dwFormatType; + DWORD dwSize, dwEncoding, dwContentType, dwFormatType, dwSignerInfoSize = 0; PCMSG_SIGNER_INFO pSignerInfo = NULL; - DWORD dwSignerInfo = 0; + // TODO: Do we really need CertInfo? Or can we just reference pSignerInfo? CERT_INFO CertInfo = { 0 }; SPROG_PUBLISHERINFO ProgPubInfo = { 0 }; wchar_t *szFileName; @@ -311,21 +311,21 @@ char* GetSignatureName(const char* path, const char* country_code, BOOL bSilent) goto out; // Get signer information size. - r = CryptMsgGetParam(hMsg, CMSG_SIGNER_INFO_PARAM, 0, NULL, &dwSignerInfo); + r = CryptMsgGetParam(hMsg, CMSG_SIGNER_INFO_PARAM, 0, NULL, &dwSignerInfoSize); if (!r) { uprintf("PKI: Failed to get signer size: %s", WinPKIErrorString()); goto out; } // Allocate memory for signer information. - pSignerInfo = (PCMSG_SIGNER_INFO)calloc(dwSignerInfo, 1); + pSignerInfo = (PCMSG_SIGNER_INFO)calloc(dwSignerInfoSize, 1); if (!pSignerInfo) { uprintf("PKI: Could not allocate memory for signer information"); goto out; } // Get Signer Information. - r = CryptMsgGetParam(hMsg, CMSG_SIGNER_INFO_PARAM, 0, (PVOID)pSignerInfo, &dwSignerInfo); + r = CryptMsgGetParam(hMsg, CMSG_SIGNER_INFO_PARAM, 0, (PVOID)pSignerInfo, &dwSignerInfoSize); if (!r) { uprintf("PKI: Failed to get signer information: %s", WinPKIErrorString()); goto out; @@ -334,10 +334,9 @@ char* GetSignatureName(const char* path, const char* country_code, BOOL bSilent) // Search for the signer certificate in the temporary certificate store. CertInfo.Issuer = pSignerInfo->Issuer; CertInfo.SerialNumber = pSignerInfo->SerialNumber; - pCertContext = CertFindCertificateInStore(hStore, ENCODING, 0, CERT_FIND_SUBJECT_CERT, (PVOID)&CertInfo, NULL); if (!pCertContext) { - uprintf("PKI: Failed to locate signer certificate in temporary store: %s", WinPKIErrorString()); + uprintf("PKI: Failed to locate signer certificate in store: %s", WinPKIErrorString()); goto out; } @@ -385,6 +384,105 @@ char* GetSignatureName(const char* path, const char* country_code, BOOL bSilent) return p; } +// Return the issuer certificate's name and thumbprint +BOOL GetIssuerCertificateInfo(uint8_t* cert, cert_info_t* info) +{ + BOOL ret = FALSE; + DWORD dwSize, dwEncoding, dwContentType, dwFormatType, dwSignerInfoSize = 0; + WIN_CERTIFICATE* pWinCert = (WIN_CERTIFICATE*)cert; + CRYPT_DATA_BLOB signedDataBlob; + HCERTSTORE hStore = NULL; + HCRYPTMSG hMsg = NULL; + PCMSG_SIGNER_INFO pSignerInfo = NULL; + PCCERT_CONTEXT pCertContext = NULL, pIssuerCertContext = NULL; + PCCERT_CHAIN_CONTEXT pChainContext = NULL; + CERT_CHAIN_PARA chainPara; + + if (cert == NULL || info == NULL || pWinCert->dwLength == 0) + return FALSE; + + signedDataBlob.cbData = pWinCert->dwLength - sizeof(WIN_CERTIFICATE); + signedDataBlob.pbData = pWinCert->bCertificate; + + // Get message handle and store handle from the signed file. + if (!CryptQueryObject(CERT_QUERY_OBJECT_BLOB, &signedDataBlob, + CERT_QUERY_CONTENT_FLAG_PKCS7_SIGNED, CERT_QUERY_FORMAT_FLAG_BINARY, + 0, &dwEncoding, &dwContentType, &dwFormatType, &hStore, &hMsg, NULL)) { + uprintf("PKI: Failed to get signature: %s", WinPKIErrorString()); + goto out; + } + + // Get signer information size. + if (!CryptMsgGetParam(hMsg, CMSG_SIGNER_CERT_INFO_PARAM, 0, NULL, &dwSignerInfoSize)) { + uprintf("PKI: Failed to get signer size: %s", WinPKIErrorString()); + goto out; + } + + // Allocate memory for signer information. + pSignerInfo = (PCMSG_SIGNER_INFO)calloc(dwSignerInfoSize, 1); + if (!pSignerInfo) { + uprintf("PKI: Could not allocate memory for signer information"); + goto out; + } + + // Get Signer Information. + if (!CryptMsgGetParam(hMsg, CMSG_SIGNER_CERT_INFO_PARAM, 0, pSignerInfo, &dwSignerInfoSize)) { + uprintf("PKI: Failed to get signer info: %s", WinPKIErrorString()); + goto out; + } + + // Search for the signer certificate in the temporary certificate store. + pCertContext = CertFindCertificateInStore(hStore, ENCODING, 0, CERT_FIND_SUBJECT_CERT, (PVOID)pSignerInfo, NULL); + if (!pCertContext) { + uprintf("PKI: Failed to locate signer certificate in store: %s", WinPKIErrorString()); + goto out; + } + + // Build a certificate chain to get the issuer (CA) certificate. + memset(&chainPara, 0, sizeof(chainPara)); + chainPara.cbSize = sizeof(CERT_CHAIN_PARA); + if (!CertGetCertificateChain(NULL, pCertContext, NULL, hStore, &chainPara, 0, NULL, &pChainContext)) { + uprintf("PKI: Failed to build certificate chain. Error code: %s", WinPKIErrorString()); + goto out; + } + + // Get the issuer's certificate (second certificate in the chain) + if (pChainContext->cChain > 0 && pChainContext->rgpChain[0]->cElement > 1) { + pIssuerCertContext = pChainContext->rgpChain[0]->rgpElement[1]->pCertContext; + } else { + uprintf("PKI: Failed to retrieve issuer's certificate: %s", WinPKIErrorString()); + goto out; + } + + // Isolate the signing certificate subject name + dwSize = CertGetNameStringA(pIssuerCertContext, CERT_NAME_ATTR_TYPE, 0, szOID_COMMON_NAME, + info->name, sizeof(info->name)); + if (dwSize <= 1) { + uprintf("PKI: Failed to get Subject Name"); + goto out; + } + + dwSize = SHA1_HASHSIZE; + if (!CryptHashCertificate(0, CALG_SHA1, 0, pIssuerCertContext->pbCertEncoded, + pIssuerCertContext->cbCertEncoded, info->thumbprint, &dwSize)) { + uprintf("PKI: Failed to compute the thumbprint: %s", WinPKIErrorString()); + goto out; + } + ret = TRUE; + +out: + safe_free(pSignerInfo); + if (pIssuerCertContext != NULL) + CertFreeCertificateContext(pIssuerCertContext); + if (pCertContext != NULL) + CertFreeCertificateContext(pCertContext); + if (hStore != NULL) + CertCloseStore(hStore, 0); + if (hMsg != NULL) + CryptMsgClose(hMsg); + return ret; +} + // The timestamping authorities we use are RFC 3161 compliant static uint64_t GetRFC3161TimeStamp(PCMSG_SIGNER_INFO pSignerInfo) { @@ -523,9 +621,8 @@ uint64_t GetSignatureTimeStamp(const char* path) HMODULE hm; HCERTSTORE hStore = NULL; HCRYPTMSG hMsg = NULL; - DWORD dwSize, dwEncoding, dwContentType, dwFormatType; + DWORD dwSize, dwEncoding, dwContentType, dwFormatType, dwSignerInfoSize = 0; PCMSG_SIGNER_INFO pSignerInfo = NULL; - DWORD dwSignerInfo = 0; wchar_t *szFileName; uint64_t timestamp = 0ULL, nested_timestamp; @@ -559,21 +656,21 @@ uint64_t GetSignatureTimeStamp(const char* path) } // Get signer information size. - r = CryptMsgGetParam(hMsg, CMSG_SIGNER_INFO_PARAM, 0, NULL, &dwSignerInfo); + r = CryptMsgGetParam(hMsg, CMSG_SIGNER_INFO_PARAM, 0, NULL, &dwSignerInfoSize); if (!r) { uprintf("PKI: Failed to get signer size: %s", WinPKIErrorString()); goto out; } // Allocate memory for signer information. - pSignerInfo = (PCMSG_SIGNER_INFO)calloc(dwSignerInfo, 1); + pSignerInfo = (PCMSG_SIGNER_INFO)calloc(dwSignerInfoSize, 1); if (!pSignerInfo) { uprintf("PKI: Could not allocate memory for signer information"); goto out; } // Get Signer Information. - r = CryptMsgGetParam(hMsg, CMSG_SIGNER_INFO_PARAM, 0, (PVOID)pSignerInfo, &dwSignerInfo); + r = CryptMsgGetParam(hMsg, CMSG_SIGNER_INFO_PARAM, 0, (PVOID)pSignerInfo, &dwSignerInfoSize); if (!r) { uprintf("PKI: Failed to get signer information: %s", WinPKIErrorString()); goto out; diff --git a/src/rufus.c b/src/rufus.c index 8e0837342a6..09dcc29fc2c 100755 --- a/src/rufus.c +++ b/src/rufus.c @@ -1610,7 +1610,7 @@ static DWORD WINAPI BootCheckThread(LPVOID param) // Check UEFI bootloaders for revocation if (IS_EFI_BOOTABLE(img_report)) { for (i = 0; i < ARRAYSIZE(img_report.efi_boot_path) && img_report.efi_boot_path[i][0] != 0; i++) { - static const char* revocation_type[] = { "UEFI DBX", "Windows SSP", "Linux SBAT", "Windows SVN" }; + static const char* revocation_type[] = { "UEFI DBX", "Windows SSP", "Linux SBAT", "Windows SVN", "Cert DBX" }; len = ReadISOFileToBuffer(image_path, img_report.efi_boot_path[i], &buf); if (len == 0) { uprintf("Warning: Failed to extract '%s' to check for UEFI revocation", img_report.efi_boot_path[i]); diff --git a/src/rufus.h b/src/rufus.h index 9240784a23e..61e26ff8936 100644 --- a/src/rufus.h +++ b/src/rufus.h @@ -528,6 +528,12 @@ typedef struct ALIGNED(64) { uint64_t bytecount; } HASH_CONTEXT; +/* Certificate info */ +typedef struct { + char name[256]; + uint8_t thumbprint[SHA1_HASHSIZE]; +} cert_info_t; + /* Hash functions */ typedef void hash_init_t(HASH_CONTEXT* ctx); typedef void hash_write_t(HASH_CONTEXT* ctx, const uint8_t* buf, size_t len); @@ -658,7 +664,7 @@ typedef struct htab_table { uint32_t size; uint32_t filled; } htab_table; -#define HTAB_EMPTY {NULL, 0, 0} +#define HTAB_EMPTY { NULL, 0, 0 } extern BOOL htab_create(uint32_t nel, htab_table* htab); extern void htab_destroy(htab_table* htab); extern uint32_t htab_hash(char* str, htab_table* htab); @@ -799,6 +805,7 @@ extern void* get_data_from_asn1(const uint8_t* buf, size_t buf_len, const char* extern int sanitize_label(char* label); extern int IsHDD(DWORD DriveIndex, uint16_t vid, uint16_t pid, const char* strid); extern char* GetSignatureName(const char* path, const char* country_code, BOOL bSilent); +extern BOOL GetIssuerCertificateInfo(uint8_t* cert, cert_info_t* info); extern uint64_t GetSignatureTimeStamp(const char* path); extern LONG ValidateSignature(HWND hDlg, const char* path); extern BOOL ValidateOpensslSignature(BYTE* pbBuffer, DWORD dwBufferLen, BYTE* pbSignature, DWORD dwSigLen); @@ -841,6 +848,7 @@ extern uint32_t ResolveDllAddress(dll_resolver_t* resolver); extern sbat_entry_t* GetSbatEntries(char* sbatlevel); extern uint16_t GetPeArch(uint8_t* buf); extern uint8_t* GetPeSection(uint8_t* buf, const char* name, uint32_t* len); +extern uint8_t* GetPeSignatureData(uint8_t* buf); extern uint8_t* RvaToPhysical(uint8_t* buf, uint32_t rva); extern uint32_t FindResourceRva(const wchar_t* name, uint8_t* root, uint8_t* dir, uint32_t* len); extern DWORD ListDirectoryContent(StrArray* arr, char* dir, uint8_t type); diff --git a/src/rufus.rc b/src/rufus.rc index 7236b2c2e6c..ac3546960ff 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2201" +CAPTION "Rufus 4.6.2202" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -399,8 +399,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2201,0 - PRODUCTVERSION 4,6,2201,0 + FILEVERSION 4,6,2202,0 + PRODUCTVERSION 4,6,2202,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -418,13 +418,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2201" + VALUE "FileVersion", "4.6.2202" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", " 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2201" + VALUE "ProductVersion", "4.6.2202" END END BLOCK "VarFileInfo" diff --git a/src/wue.c b/src/wue.c index 46db5bfd5be..d7b14cf4e4e 100644 --- a/src/wue.c +++ b/src/wue.c @@ -1026,7 +1026,7 @@ BOOL ApplyWindowsCustomization(char drive_letter, int flags) } // Now that we should be able to write to the destination directories, copy the content. ListDirectoryContent(&files, path, LIST_DIR_TYPE_FILE | LIST_DIR_TYPE_RECURSIVE); - for (i = 0; r && i < (int)files.Index; i++) { + for (i = 0; i < (int)files.Index; i++) { rep = remove_substr(files.String[i], "_EX"); assert(rep != NULL); TakeOwnership(rep); From ede52c57e65ff25d9b27591eb233705ced62a30d Mon Sep 17 00:00:00 2001 From: Pete Batard <pete@akeo.ie> Date: Wed, 9 Oct 2024 20:18:42 +0100 Subject: [PATCH 21/30] [misc] improve revoked UEFI bootloader reporting * Also fix SBAT not being properly parsed for PE32 executables. * Also fix signature truncation in GetIssuerCertificateInfo() and fall back to returning signer data if issuer is not available (which is typically the case for GRUB signed bootloaders). * Also fix status messages on user cancellation/proceeding. --- res/loc/rufus.loc | 1 + src/hash.c | 33 +++++++++++++++++------------ src/parser.c | 24 +++++++++++++-------- src/pki.c | 54 ++++++++++++++++++++++++----------------------- src/rufus.c | 45 +++++++++++++++++++++++++-------------- src/rufus.h | 2 +- src/rufus.rc | 10 ++++----- 7 files changed, 98 insertions(+), 71 deletions(-) diff --git a/res/loc/rufus.loc b/res/loc/rufus.loc index 327443281b0..63d590fd2dd 100644 --- a/res/loc/rufus.loc +++ b/res/loc/rufus.loc @@ -610,6 +610,7 @@ t MSG_347 "Expert Mode" t MSG_348 "Extracting archive files: %s" t MSG_349 "Use Rufus MBR" t MSG_350 "Use 'Windows UEFI CA 2023' signed bootloaders [EXPERIMENTAL]" +t MSG_351 "Checking for UEFI bootloader revocation..." # The following messages are for the Windows Store listing only and are not used by the application t MSG_900 "Rufus is a utility that helps format and create bootable USB flash drives, such as USB keys/pendrives, memory sticks, etc." t MSG_901 "Official site: %s" diff --git a/src/hash.c b/src/hash.c index 5ff7df1061b..1b0b2ce9f19 100644 --- a/src/hash.c +++ b/src/hash.c @@ -117,7 +117,7 @@ StrArray modified_files = { 0 }; extern int default_thread_priority; extern const char* efi_archname[ARCH_MAX]; extern char* sbat_level_txt; -extern BOOL expert_mode; +extern BOOL expert_mode, usb_debug; /* * Rotate 32 or 64 bit integers by n bytes. @@ -2081,7 +2081,7 @@ BOOL IsFileInDB(const char* path) return FALSE; } -BOOL IsRevokedBySbat(uint8_t* buf, uint32_t len) +static BOOL IsRevokedBySbat(uint8_t* buf, uint32_t len) { char* sbat = NULL, *version_str; uint32_t i, j, sbat_len; @@ -2109,6 +2109,7 @@ BOOL IsRevokedBySbat(uint8_t* buf, uint32_t len) for (; sbat[i] != ',' && sbat[i] != '\0' && sbat[i] != '\n'; i++); sbat[i++] = '\0'; entry.version = atoi(version_str); + uuprintf(" SBAT: %s,%d", entry.product, entry.version); for (; sbat[i] != '\0' && sbat[i] != '\n'; i++); if (entry.version == 0) continue; @@ -2121,13 +2122,13 @@ BOOL IsRevokedBySbat(uint8_t* buf, uint32_t len) return FALSE; } -BOOL IsRevokedBySvn(uint8_t* buf, uint32_t len) +static BOOL IsRevokedBySvn(uint8_t* buf, uint32_t len) { wchar_t* rsrc_name = NULL; uint8_t *root; uint32_t i, j, rsrc_rva, rsrc_len, *svn_ver; IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; - IMAGE_NT_HEADERS* pe_header; + IMAGE_NT_HEADERS32* pe_header; IMAGE_NT_HEADERS64* pe64_header; IMAGE_DATA_DIRECTORY img_data_dir; @@ -2143,7 +2144,7 @@ BOOL IsRevokedBySvn(uint8_t* buf, uint32_t len) if (rsrc_name == NULL) continue; - pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + pe_header = (IMAGE_NT_HEADERS32*)&buf[dos_header->e_lfanew]; if (pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_I386 || pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_ARM) { img_data_dir = pe_header->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_RESOURCE]; } else { @@ -2157,6 +2158,7 @@ BOOL IsRevokedBySvn(uint8_t* buf, uint32_t len) if (rsrc_rva != 0) { if (rsrc_len == sizeof(uint32_t)) { svn_ver = (uint32_t*)RvaToPhysical(buf, rsrc_rva); + uuprintf(" SVN version: %d.%d", *svn_ver >> 16, *svn_ver & 0xffff); if (svn_ver != NULL && *svn_ver < sbat_entries[i].version) return TRUE; } else { @@ -2167,18 +2169,20 @@ BOOL IsRevokedBySvn(uint8_t* buf, uint32_t len) return FALSE; } -BOOL IsRevokedByCert(uint8_t* buf, uint32_t len) +static BOOL IsRevokedByCert(uint8_t* buf, uint32_t len) { int i; uint8_t* cert; cert_info_t info; cert = GetPeSignatureData(buf); - if (cert == NULL) - return FALSE; - if (!GetIssuerCertificateInfo(cert, &info)) + i = GetIssuerCertificateInfo(cert, &info); + if (i == 0) + uuprintf(" (Unsigned Bootloader)"); + if (i <= 0) return FALSE; + uuprintf(" Signed by: %s", info.name); for (i = 0; i < ARRAYSIZE(certdbx); i += SHA1_HASHSIZE) { if (!expert_mode) continue; @@ -2195,7 +2199,7 @@ int IsBootloaderRevoked(uint8_t* buf, uint32_t len) uint32_t i; uint8_t hash[SHA256_HASHSIZE]; IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; - IMAGE_NT_HEADERS* pe_header; + IMAGE_NT_HEADERS32* pe_header; // Fall back to embedded sbat_level.txt if we couldn't access remote if (sbat_entries == NULL) { @@ -2205,11 +2209,15 @@ int IsBootloaderRevoked(uint8_t* buf, uint32_t len) if (buf == NULL || len < 0x100 || dos_header->e_magic != IMAGE_DOS_SIGNATURE) return -2; - pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + pe_header = (IMAGE_NT_HEADERS32*)&buf[dos_header->e_lfanew]; if (pe_header->Signature != IMAGE_NT_SIGNATURE) return -2; if (!PE256Buffer(buf, len, hash)) return -1; + // Check for UEFI DBX certificate revocation + // This also displays the name of the signer/issuer cert if available + if (IsRevokedByCert(buf, len)) + return 5; // Check for UEFI DBX revocation for (i = 0; i < ARRAYSIZE(pe256dbx); i += SHA256_HASHSIZE) if (memcmp(hash, &pe256dbx[i], SHA256_HASHSIZE) == 0) @@ -2224,9 +2232,6 @@ int IsBootloaderRevoked(uint8_t* buf, uint32_t len) // Check for Microsoft SVN revocation if (IsRevokedBySvn(buf, len)) return 4; - // Check for DBX certificate revocation - if (IsRevokedByCert(buf, len)) - return 5; return 0; } diff --git a/src/parser.c b/src/parser.c index 7bc21692ddf..4c5828268c4 100644 --- a/src/parser.c +++ b/src/parser.c @@ -1643,12 +1643,12 @@ sbat_entry_t* GetSbatEntries(char* sbatlevel) uint16_t GetPeArch(uint8_t* buf) { IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; - IMAGE_NT_HEADERS* pe_header; + IMAGE_NT_HEADERS32* pe_header; if (buf == NULL || dos_header->e_magic != IMAGE_DOS_SIGNATURE) return IMAGE_FILE_MACHINE_UNKNOWN; - pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + pe_header = (IMAGE_NT_HEADERS32*)&buf[dos_header->e_lfanew]; if (pe_header->Signature != IMAGE_NT_SIGNATURE) return IMAGE_FILE_MACHINE_UNKNOWN; return pe_header->FileHeader.Machine; @@ -1660,7 +1660,7 @@ uint8_t* GetPeSection(uint8_t* buf, const char* name, uint32_t* len) char section_name[IMAGE_SIZEOF_SHORT_NAME] = { 0 }; uint32_t i, nb_sections; IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; - IMAGE_NT_HEADERS* pe_header; + IMAGE_NT_HEADERS32* pe_header; IMAGE_NT_HEADERS64* pe64_header; IMAGE_SECTION_HEADER* section_header; @@ -1669,7 +1669,7 @@ uint8_t* GetPeSection(uint8_t* buf, const char* name, uint32_t* len) if (buf == NULL || name == NULL || dos_header->e_magic != IMAGE_DOS_SIGNATURE) return NULL; - pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + pe_header = (IMAGE_NT_HEADERS32*)&buf[dos_header->e_lfanew]; if (pe_header->Signature != IMAGE_NT_SIGNATURE) return NULL; if (pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_I386 || pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_ARM) { @@ -1695,14 +1695,14 @@ uint8_t* RvaToPhysical(uint8_t* buf, uint32_t rva) { uint32_t i, nb_sections; IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; - IMAGE_NT_HEADERS* pe_header; + IMAGE_NT_HEADERS32* pe_header; IMAGE_NT_HEADERS64* pe64_header; IMAGE_SECTION_HEADER* section_header; if (buf == NULL || dos_header->e_magic != IMAGE_DOS_SIGNATURE) return NULL; - pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + pe_header = (IMAGE_NT_HEADERS32*)&buf[dos_header->e_lfanew]; if (pe_header->Signature != IMAGE_NT_SIGNATURE) return NULL; if (pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_I386 || pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_ARM) { @@ -1767,18 +1767,24 @@ uint32_t FindResourceRva(const wchar_t* name, uint8_t* root, uint8_t* dir, uint3 uint8_t* GetPeSignatureData(uint8_t* buf) { IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; - IMAGE_NT_HEADERS* pe_header; + IMAGE_NT_HEADERS32* pe_header; + IMAGE_NT_HEADERS64* pe64_header; IMAGE_DATA_DIRECTORY sec_dir; WIN_CERTIFICATE* cert; if (buf == NULL || dos_header->e_magic != IMAGE_DOS_SIGNATURE) return NULL; - pe_header = (IMAGE_NT_HEADERS*)&buf[dos_header->e_lfanew]; + pe_header = (IMAGE_NT_HEADERS32*)&buf[dos_header->e_lfanew]; if (pe_header->Signature != IMAGE_NT_SIGNATURE) return NULL; - sec_dir = pe_header->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_SECURITY]; + if (pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_I386 || pe_header->FileHeader.Machine == IMAGE_FILE_MACHINE_ARM) { + sec_dir = pe_header->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_SECURITY]; + } else { + pe64_header = (IMAGE_NT_HEADERS64*)pe_header; + sec_dir = pe64_header->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_SECURITY]; + } if (sec_dir.VirtualAddress == 0 || sec_dir.Size == 0) return NULL; diff --git a/src/pki.c b/src/pki.c index 09dda43c80f..fe923c15376 100644 --- a/src/pki.c +++ b/src/pki.c @@ -384,27 +384,31 @@ char* GetSignatureName(const char* path, const char* country_code, BOOL bSilent) return p; } -// Return the issuer certificate's name and thumbprint -BOOL GetIssuerCertificateInfo(uint8_t* cert, cert_info_t* info) +// Fills the certificate's name and thumbprint. +// Tries the issuer first, and if none is available, falls back to current cert. +// Returns 0 for unsigned, -1 on error, 1 for signer or 2 for issuer. +int GetIssuerCertificateInfo(uint8_t* cert, cert_info_t* info) { - BOOL ret = FALSE; + int ret = 0; DWORD dwSize, dwEncoding, dwContentType, dwFormatType, dwSignerInfoSize = 0; WIN_CERTIFICATE* pWinCert = (WIN_CERTIFICATE*)cert; CRYPT_DATA_BLOB signedDataBlob; HCERTSTORE hStore = NULL; HCRYPTMSG hMsg = NULL; PCMSG_SIGNER_INFO pSignerInfo = NULL; - PCCERT_CONTEXT pCertContext = NULL, pIssuerCertContext = NULL; + PCCERT_CONTEXT pCertContext[2] = { NULL, NULL }; PCCERT_CHAIN_CONTEXT pChainContext = NULL; CERT_CHAIN_PARA chainPara; + int CertIndex = 0; - if (cert == NULL || info == NULL || pWinCert->dwLength == 0) - return FALSE; - - signedDataBlob.cbData = pWinCert->dwLength - sizeof(WIN_CERTIFICATE); - signedDataBlob.pbData = pWinCert->bCertificate; + if (info == NULL) + return -1; + if (pWinCert == NULL || pWinCert->dwLength == 0) + return 0; // Get message handle and store handle from the signed file. + signedDataBlob.cbData = pWinCert->dwLength; + signedDataBlob.pbData = pWinCert->bCertificate; if (!CryptQueryObject(CERT_QUERY_OBJECT_BLOB, &signedDataBlob, CERT_QUERY_CONTENT_FLAG_PKCS7_SIGNED, CERT_QUERY_FORMAT_FLAG_BINARY, 0, &dwEncoding, &dwContentType, &dwFormatType, &hStore, &hMsg, NULL)) { @@ -420,7 +424,7 @@ BOOL GetIssuerCertificateInfo(uint8_t* cert, cert_info_t* info) // Allocate memory for signer information. pSignerInfo = (PCMSG_SIGNER_INFO)calloc(dwSignerInfoSize, 1); - if (!pSignerInfo) { + if (pSignerInfo == NULL) { uprintf("PKI: Could not allocate memory for signer information"); goto out; } @@ -432,8 +436,8 @@ BOOL GetIssuerCertificateInfo(uint8_t* cert, cert_info_t* info) } // Search for the signer certificate in the temporary certificate store. - pCertContext = CertFindCertificateInStore(hStore, ENCODING, 0, CERT_FIND_SUBJECT_CERT, (PVOID)pSignerInfo, NULL); - if (!pCertContext) { + pCertContext[0] = CertFindCertificateInStore(hStore, ENCODING, 0, CERT_FIND_SUBJECT_CERT, (PVOID)pSignerInfo, NULL); + if (pCertContext[0] == NULL) { uprintf("PKI: Failed to locate signer certificate in store: %s", WinPKIErrorString()); goto out; } @@ -441,21 +445,19 @@ BOOL GetIssuerCertificateInfo(uint8_t* cert, cert_info_t* info) // Build a certificate chain to get the issuer (CA) certificate. memset(&chainPara, 0, sizeof(chainPara)); chainPara.cbSize = sizeof(CERT_CHAIN_PARA); - if (!CertGetCertificateChain(NULL, pCertContext, NULL, hStore, &chainPara, 0, NULL, &pChainContext)) { + if (!CertGetCertificateChain(NULL, pCertContext[0], NULL, hStore, &chainPara, 0, NULL, &pChainContext)) { uprintf("PKI: Failed to build certificate chain. Error code: %s", WinPKIErrorString()); goto out; } - // Get the issuer's certificate (second certificate in the chain) + // Try to get the issuer's certificate (second certificate in the chain) if available if (pChainContext->cChain > 0 && pChainContext->rgpChain[0]->cElement > 1) { - pIssuerCertContext = pChainContext->rgpChain[0]->rgpElement[1]->pCertContext; - } else { - uprintf("PKI: Failed to retrieve issuer's certificate: %s", WinPKIErrorString()); - goto out; + pCertContext[1] = pChainContext->rgpChain[0]->rgpElement[1]->pCertContext; + CertIndex = 1; } // Isolate the signing certificate subject name - dwSize = CertGetNameStringA(pIssuerCertContext, CERT_NAME_ATTR_TYPE, 0, szOID_COMMON_NAME, + dwSize = CertGetNameStringA(pCertContext[CertIndex], CERT_NAME_ATTR_TYPE, 0, szOID_COMMON_NAME, info->name, sizeof(info->name)); if (dwSize <= 1) { uprintf("PKI: Failed to get Subject Name"); @@ -463,19 +465,19 @@ BOOL GetIssuerCertificateInfo(uint8_t* cert, cert_info_t* info) } dwSize = SHA1_HASHSIZE; - if (!CryptHashCertificate(0, CALG_SHA1, 0, pIssuerCertContext->pbCertEncoded, - pIssuerCertContext->cbCertEncoded, info->thumbprint, &dwSize)) { + if (!CryptHashCertificate(0, CALG_SHA1, 0, pCertContext[CertIndex]->pbCertEncoded, + pCertContext[CertIndex]->cbCertEncoded, info->thumbprint, &dwSize)) { uprintf("PKI: Failed to compute the thumbprint: %s", WinPKIErrorString()); goto out; } - ret = TRUE; + ret = CertIndex + 1; out: safe_free(pSignerInfo); - if (pIssuerCertContext != NULL) - CertFreeCertificateContext(pIssuerCertContext); - if (pCertContext != NULL) - CertFreeCertificateContext(pCertContext); + if (pCertContext[1] != NULL) + CertFreeCertificateContext(pCertContext[1]); + if (pCertContext[0] != NULL) + CertFreeCertificateContext(pCertContext[0]); if (hStore != NULL) CertCloseStore(hStore, 0); if (hMsg != NULL) diff --git a/src/rufus.c b/src/rufus.c index 09dcc29fc2c..2f317ae5160 100755 --- a/src/rufus.c +++ b/src/rufus.c @@ -1399,7 +1399,7 @@ DWORD WINAPI ImageScanThread(LPVOID param) // Likewise, boot check will block message processing => use a thread static DWORD WINAPI BootCheckThread(LPVOID param) { - int i, r, username_index = -1; + int i, r, rr, username_index = -1; FILE *fd; uint32_t len; uint8_t* buf = NULL; @@ -1609,33 +1609,43 @@ static DWORD WINAPI BootCheckThread(LPVOID param) // Check UEFI bootloaders for revocation if (IS_EFI_BOOTABLE(img_report)) { + assert(ARRAYSIZE(img_report.efi_boot_path) > 0); + PrintStatus(0, MSG_351); + uuprintf("UEFI Secure Boot revocation checks:"); + rr = 0; for (i = 0; i < ARRAYSIZE(img_report.efi_boot_path) && img_report.efi_boot_path[i][0] != 0; i++) { static const char* revocation_type[] = { "UEFI DBX", "Windows SSP", "Linux SBAT", "Windows SVN", "Cert DBX" }; + cdio_loglevel_default = CDIO_LOG_WARN; len = ReadISOFileToBuffer(image_path, img_report.efi_boot_path[i], &buf); + cdio_loglevel_default = usb_debug ? CDIO_LOG_DEBUG : CDIO_LOG_WARN; if (len == 0) { uprintf("Warning: Failed to extract '%s' to check for UEFI revocation", img_report.efi_boot_path[i]); continue; } + uuprintf("• %s", img_report.efi_boot_path[i]); r = IsBootloaderRevoked(buf, len); safe_free(buf); if (r > 0) { assert(r <= ARRAYSIZE(revocation_type)); + if (rr == 0) + rr = r; uprintf("Warning: '%s' has been revoked by %s", img_report.efi_boot_path[i], revocation_type[r - 1]); is_bootloader_revoked = TRUE; - switch (r) { - case 2: - msg = lmprintf(MSG_341, "Error code: 0xc0000428"); - break; - default: - msg = lmprintf(MSG_340); - break; - } - r = MessageBoxExU(hMainDialog, lmprintf(MSG_339, msg), lmprintf(MSG_338), - MB_OKCANCEL | MB_ICONWARNING | MB_IS_RTL, selected_langid); - if (r == IDCANCEL) - goto out; + } + } + if (rr > 0) { + switch (rr) { + case 2: + msg = lmprintf(MSG_341, "Error code: 0xc0000428"); + break; + default: + msg = lmprintf(MSG_340); break; } + r = MessageBoxExU(hMainDialog, lmprintf(MSG_339, msg), lmprintf(MSG_338), + MB_OKCANCEL | MB_ICONWARNING | MB_IS_RTL, selected_langid); + if (r == IDCANCEL) + goto out; } } @@ -1691,7 +1701,6 @@ static DWORD WINAPI BootCheckThread(LPVOID param) IGNORE_RETVAL(_chdir(tmp)); // The following loops through the grub2 version (which may have the ISO label appended) // and breaks it according to '.' or '-' until it finds a match on the server. - // for (i = (int)strlen(img_report.grub2_version), grub2_len = 0; i > 0 && grub2_len <= 0; i--) { c = img_report.grub2_version[i]; if (c != 0 && c != '.' && c != '-') @@ -1761,7 +1770,7 @@ static DWORD WINAPI BootCheckThread(LPVOID param) IGNORE_RETVAL(_chdirU(app_data_dir)); IGNORE_RETVAL(_mkdir(FILES_DIR)); IGNORE_RETVAL(_chdir(FILES_DIR)); - for (i=0; i<2; i++) { + for (i = 0; i < 2; i++) { // Check if we already have the relevant ldlinux_v#.##.sys & ldlinux_v#.##.bss files static_sprintf(tmp, "%s-%s%s\\%s.%s", syslinux, img_report.sl_version_str, img_report.sl_version_ext, ldlinux, ldlinux_ext[i]); @@ -1783,7 +1792,7 @@ static DWORD WINAPI BootCheckThread(LPVOID param) lmprintf(MSG_115), MB_YESNO | MB_ICONWARNING | MB_IS_RTL, selected_langid); if (r != IDYES) goto out; - for (i=0; i<2; i++) { + for (i = 0; i < 2; i++) { static_sprintf(tmp, "%s-%s", syslinux, img_report.sl_version_str); IGNORE_RETVAL(_mkdir(tmp)); if (*img_report.sl_version_ext != 0) { @@ -2970,6 +2979,7 @@ static INT_PTR CALLBACK MainCallback(HWND hDlg, UINT message, WPARAM wParam, LPA if (!Notification(MSG_WARNING_QUESTION, NULL, NULL, title, lmprintf(MSG_132))) goto aborted_start; } + PrintStatus(0, MSG_142); GetWindowTextU(hDeviceList, tmp, ARRAYSIZE(tmp)); if (MessageBoxExU(hMainDialog, lmprintf(MSG_003, tmp), @@ -3005,6 +3015,9 @@ static INT_PTR CALLBACK MainCallback(HWND hDlg, UINT message, WPARAM wParam, LPA if (queued_hotplug_event) SendMessage(hDlg, UM_MEDIA_CHANGE, 0, 0); if (wParam == BOOTCHECK_CANCEL) { + nb_devices = ComboBox_GetCount(hDeviceList); + PrintStatus(0, (nb_devices == 1) ? MSG_208 : MSG_209, nb_devices); + PrintStatus(5000, MSG_041); if (unattend_xml_path != NULL) { DeleteFileU(unattend_xml_path); unattend_xml_path = NULL; diff --git a/src/rufus.h b/src/rufus.h index 61e26ff8936..1d5a7adba34 100644 --- a/src/rufus.h +++ b/src/rufus.h @@ -805,7 +805,7 @@ extern void* get_data_from_asn1(const uint8_t* buf, size_t buf_len, const char* extern int sanitize_label(char* label); extern int IsHDD(DWORD DriveIndex, uint16_t vid, uint16_t pid, const char* strid); extern char* GetSignatureName(const char* path, const char* country_code, BOOL bSilent); -extern BOOL GetIssuerCertificateInfo(uint8_t* cert, cert_info_t* info); +extern int GetIssuerCertificateInfo(uint8_t* cert, cert_info_t* info); extern uint64_t GetSignatureTimeStamp(const char* path); extern LONG ValidateSignature(HWND hDlg, const char* path); extern BOOL ValidateOpensslSignature(BYTE* pbBuffer, DWORD dwBufferLen, BYTE* pbSignature, DWORD dwSigLen); diff --git a/src/rufus.rc b/src/rufus.rc index ac3546960ff..a810002702c 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2202" +CAPTION "Rufus 4.6.2203" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -399,8 +399,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2202,0 - PRODUCTVERSION 4,6,2202,0 + FILEVERSION 4,6,2203,0 + PRODUCTVERSION 4,6,2203,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -418,13 +418,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2202" + VALUE "FileVersion", "4.6.2203" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", " 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2202" + VALUE "ProductVersion", "4.6.2203" END END BLOCK "VarFileInfo" From 5439ca8a8325bec0bb2ac015d6ada9005d6d30ad Mon Sep 17 00:00:00 2001 From: Pete Batard <pete@akeo.ie> Date: Thu, 10 Oct 2024 13:04:58 +0100 Subject: [PATCH 22/30] [md5sum] fix md5sum computation for the setup wrapper * Also fix a couple small memory leaks and potential NULL deref. * Also report saved path when saving to image. --- src/hash.c | 18 ++++++++---------- src/iso.c | 2 +- src/rufus.c | 2 ++ src/rufus.h | 1 + src/rufus.rc | 10 +++++----- src/vhd.c | 10 +++++++--- src/wue.c | 22 ++++++++++++++++++---- 7 files changed, 42 insertions(+), 23 deletions(-) diff --git a/src/hash.c b/src/hash.c index 1b0b2ce9f19..ee21cc51e1f 100644 --- a/src/hash.c +++ b/src/hash.c @@ -2158,9 +2158,11 @@ static BOOL IsRevokedBySvn(uint8_t* buf, uint32_t len) if (rsrc_rva != 0) { if (rsrc_len == sizeof(uint32_t)) { svn_ver = (uint32_t*)RvaToPhysical(buf, rsrc_rva); - uuprintf(" SVN version: %d.%d", *svn_ver >> 16, *svn_ver & 0xffff); - if (svn_ver != NULL && *svn_ver < sbat_entries[i].version) - return TRUE; + if (svn_ver != NULL) { + uuprintf(" SVN version: %d.%d", *svn_ver >> 16, *svn_ver & 0xffff); + if (*svn_ver < sbat_entries[i].version) + return TRUE; + } } else { uprintf("WARNING: Unexpected Secure Version Number size"); } @@ -2264,12 +2266,12 @@ void UpdateMD5Sum(const char* dest_dir, const char* md5sum_name) char *md5_data = NULL, *new_data = NULL, *str_pos, *d, *s, *p; if (!img_report.has_md5sum && !validate_md5sum) - goto out; + return; static_sprintf(md5_path, "%s\\%s", dest_dir, md5sum_name); md5_size = read_file(md5_path, (uint8_t**)&md5_data); if (md5_size == 0) - goto out; + return; for (i = 0; i < modified_files.Index; i++) { for (j = 0; j < (uint32_t)strlen(modified_files.String[i]); j++) @@ -2301,7 +2303,7 @@ void UpdateMD5Sum(const char* dest_dir, const char* md5sum_name) new_data = malloc(md5_size + 1024); assert(new_data != NULL); if (new_data == NULL) - goto out; + return; // Will be nonzero if we created the file, otherwise zero if (md5sum_totalbytes != 0) { snprintf(new_data, md5_size + 1024, "# md5sum_totalbytes = 0x%llx\n", md5sum_totalbytes); @@ -2372,10 +2374,6 @@ void UpdateMD5Sum(const char* dest_dir, const char* md5sum_name) write_file(md5_path, md5_data, md5_size); free(md5_data); - -out: - // We no longer need the string array at this stage - StrArrayDestroy(&modified_files); } #if defined(_DEBUG) || defined(TEST) || defined(ALPHA) diff --git a/src/iso.c b/src/iso.c index eed8c5a3d18..ab043607737 100644 --- a/src/iso.c +++ b/src/iso.c @@ -1134,7 +1134,7 @@ BOOL ExtractISO(const char* src_iso, const char* dest_dir, BOOL scan) last_nb_blocks = 0; iso_blocking_status = 0; symlinked_syslinux[0] = 0; - StrArrayCreate(&modified_files, 8); + StrArrayClear(&modified_files); if (validate_md5sum) { md5sum_totalbytes = 0; // If there isn't an already existing md5sum.txt create one diff --git a/src/rufus.c b/src/rufus.c index 2f317ae5160..0ed225bddfa 100755 --- a/src/rufus.c +++ b/src/rufus.c @@ -2103,6 +2103,7 @@ static void InitDialog(HWND hDlg) // Create the string arrays StrArrayCreate(&BlockingProcessList, 16); StrArrayCreate(&ImageList, 16); + StrArrayCreate(&modified_files, 8); // Set various checkboxes CheckDlgButton(hDlg, IDC_QUICK_FORMAT, BST_CHECKED); CheckDlgButton(hDlg, IDC_EXTENDED_LABEL, BST_CHECKED); @@ -2277,6 +2278,7 @@ static INT_PTR CALLBACK MainCallback(HWND hDlg, UINT message, WPARAM wParam, LPA StopProcessSearch(); StrArrayDestroy(&BlockingProcessList); StrArrayDestroy(&ImageList); + StrArrayDestroy(&modified_files); DestroyAllTooltips(); DestroyWindow(hLogDialog); GetWindowRect(hDlg, &relaunch_rc); diff --git a/src/rufus.h b/src/rufus.h index 1d5a7adba34..0f3d44bc911 100644 --- a/src/rufus.h +++ b/src/rufus.h @@ -710,6 +710,7 @@ extern unsigned long syslinux_ldlinux_len[2]; extern char ubuffer[UBUFFER_SIZE], embedded_sl_version_str[2][12]; extern char szFolderPath[MAX_PATH], app_dir[MAX_PATH], temp_dir[MAX_PATH], system_dir[MAX_PATH]; extern char sysnative_dir[MAX_PATH], app_data_dir[MAX_PATH], *image_path, *fido_url; +extern StrArray modified_files; /* * Shared prototypes diff --git a/src/rufus.rc b/src/rufus.rc index a810002702c..cffac279efd 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2203" +CAPTION "Rufus 4.6.2204" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -399,8 +399,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2203,0 - PRODUCTVERSION 4,6,2203,0 + FILEVERSION 4,6,2204,0 + PRODUCTVERSION 4,6,2204,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -418,13 +418,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2203" + VALUE "FileVersion", "4.6.2204" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", " 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2203" + VALUE "ProductVersion", "4.6.2204" END END BLOCK "VarFileInfo" diff --git a/src/vhd.c b/src/vhd.c index 4446d89ad49..a184b2f63e1 100644 --- a/src/vhd.c +++ b/src/vhd.c @@ -729,7 +729,7 @@ BOOL WimExtractFile(const char* image, int index, const char* src, const char* d /// <returns>TRUE if the index was found in the image, FALSE otherwise.</returns> BOOL WimIsValidIndex(const char* image, int index) { - int i = 1; + int i = 1, cur_index; BOOL r = FALSE; DWORD dw = 0; HANDLE hWim = NULL; @@ -768,7 +768,9 @@ BOOL WimIsValidIndex(const char* image, int index) goto out; while ((str = get_token_data_file_indexed("IMAGE INDEX", xml_file, i)) != NULL) { - if (atoi(str) == index) { + cur_index = atoi(str); + safe_free(str); + if (cur_index == index) { r = TRUE; break; } @@ -1061,7 +1063,7 @@ static DWORD WINAPI VhdSaveImageThread(void* param) r = 0; UpdateProgressWithInfo(OP_FORMAT, MSG_261, SelectedDrive.DiskSize, SelectedDrive.DiskSize); - uprintf("Operation complete."); + uprintf("Saved '%s'", img_save->ImagePath); out: safe_closehandle(overlapped.hEvent); @@ -1100,6 +1102,8 @@ static DWORD WINAPI FfuSaveImageThread(void* param) safe_free(img_save->DevicePath); safe_free(img_save->ImagePath); PostMessage(hMainDialog, UM_FORMAT_COMPLETED, (WPARAM)TRUE, 0); + if (!IS_ERROR(ErrorStatus)) + uprintf("Saved '%s'", img_save->ImagePath); ExitThread(r); } diff --git a/src/wue.c b/src/wue.c index d7b14cf4e4e..8281a6f4693 100644 --- a/src/wue.c +++ b/src/wue.c @@ -806,12 +806,14 @@ BOOL ApplyWindowsCustomization(char drive_letter, int flags) char appraiserres_dll_dst[] = "?:\\sources\\appraiserres.bak"; char setup_exe[] = "?:\\setup.exe"; char setup_dll[] = "?:\\setup.dll"; + char md5sum_path[] = "?:\\md5sum.txt"; char *mount_path = NULL, path[MAX_PATH]; uint8_t* buf = NULL; uint16_t setup_arch; HKEY hKey = NULL, hSubKey = NULL; LSTATUS status; DWORD dwDisp, dwVal = 1, dwSize; + FILE* fd_md5sum; assert(unattend_xml_path != NULL); uprintf("Applying Windows customization:"); @@ -854,6 +856,7 @@ BOOL ApplyWindowsCustomization(char drive_letter, int flags) if (img_report.win_version.build >= 26000) { setup_exe[0] = drive_letter; setup_dll[0] = drive_letter; + md5sum_path[0] = drive_letter; dwSize = read_file(setup_exe, &buf); if (dwSize != 0) { setup_arch = GetPeArch(buf); @@ -864,13 +867,24 @@ BOOL ApplyWindowsCustomization(char drive_letter, int flags) uprintf("Could not rename '%s': %s", setup_exe, WindowsErrorString()); } else { uprintf("Renamed '%s' → '%s'", setup_exe, setup_dll); - uprintf("Created '%s' bypass wrapper (from embedded)", setup_exe); buf = GetResource(hMainInstance, MAKEINTRESOURCEA(setup_arch == IMAGE_FILE_MACHINE_AMD64 ? IDR_SETUP_X64 : IDR_SETUP_ARM64), _RT_RCDATA, "setup.exe", &dwSize, FALSE); - if (buf == NULL) + if (buf == NULL) { uprintf("Could not access embedded 'setup.exe'"); - else - write_file(setup_exe, buf, dwSize); + } else if (write_file(setup_exe, buf, dwSize) == dwSize) { + uprintf("Created '%s' bypass wrapper (from embedded)", setup_exe); + if (validate_md5sum) { + if ((fd_md5sum = fopenU(md5sum_path, "ab")) != NULL) { + fprintf(fd_md5sum, "00000000000000000000000000000000 ./setup.dll\n"); + fclose(fd_md5sum); + } + StrArrayAdd(&modified_files, setup_exe, TRUE); + StrArrayAdd(&modified_files, setup_dll, TRUE); + md5sum_totalbytes += dwSize; + } + } else { + uprintf("Could not create '%s' bypass wrapper", setup_exe); + } } } } From 4d42b7a73a036a01bf3676852f9eb10fd9f1a16c Mon Sep 17 00:00:00 2001 From: Pete Batard <pete@akeo.ie> Date: Thu, 10 Oct 2024 13:08:33 +0100 Subject: [PATCH 23/30] [efi] further improve revoked UEFI bootloader reporting * Do not report SBAT revocations unless we actually have a formal Secure Boot signed bootloader. * Also reduce verbose log pollution by libcdio. --- ChangeLog.txt | 8 +++++ src/db.h | 19 ++++++++++-- src/hash.c | 56 +++++++++++++++++++++++---------- src/iso.c | 16 ++++++---- src/rufus.c | 86 +++++++++++++++++++++++++++++++-------------------- src/rufus.h | 15 ++++++++- src/rufus.rc | 10 +++--- 7 files changed, 145 insertions(+), 65 deletions(-) diff --git a/ChangeLog.txt b/ChangeLog.txt index 3200c139ddd..66f61081454 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -1,3 +1,11 @@ +o Version 4.6 (2024.10.??) + Add a new setup.exe wrapper to bypass Windows 11 24H2 in-place upgrade restrictions + Add TimeZone to regional options replication + Set local account passwords to not expire by default + Fix an error when trying to write compressed VHD images + Fix an error when invoking Rufus from the PowerShell commandline + Improve revoked UEFI bootloaders check to support Linux SBAT, Windows SVN and cert DBX + o Version 4.5 (2024.05.22) Add new advanced option to perform runtime UEFI media validation of suitable images (Windows, most Linux) Move the 'Use Rufus MBR' advanced option to a cheat mode (Alt-A) diff --git a/src/db.h b/src/db.h index 52c026598e2..1b4f4e85823 100644 --- a/src/db.h +++ b/src/db.h @@ -658,11 +658,26 @@ static uint8_t pe256dbx[] = { }; /* - * Contains the SHA-1 thumbprint of certificates that are being revoked by DBX. + * Contains the SHA-1 thumbprints of the issuer certificate of the official + * Secure Boot signing authority (i.e. Microsoft). + */ +static uint8_t certauth[] = { + // 'Microsoft Windows Production PCA 2011' + 0x58, 0x0a, 0x6f, 0x4c, 0xc4, 0xe4, 0xb6, 0x69, 0xb9, 0xeb, 0xdc, 0x1b, 0x2b, 0x3e, 0x08, 0x7b, 0x80, 0xd0, 0x67, 0x8d, + // 'Microsoft Corporation UEFI CA 2011' + 0x46, 0xde, 0xf6, 0x3b, 0x5c, 0xe6, 0x1c, 0xf8, 0xba, 0x0d, 0xe2, 0xe6, 0x63, 0x9c, 0x10, 0x19, 0xd0, 0xed, 0x14, 0xf3, + // 'Windows UEFI CA 2023' + 0x45, 0xa0, 0xfa, 0x32, 0x60, 0x47, 0x73, 0xc8, 0x24, 0x33, 0xc3, 0xb7, 0xd5, 0x9e, 0x74, 0x66, 0xb3, 0xac, 0x0c, 0x67, + // 'Microsoft UEFI CA 2023' + 0xb5, 0xee, 0xb4, 0xa6, 0x70, 0x60, 0x48, 0x07, 0x3f, 0x0e, 0xd2, 0x96, 0xe7, 0xf5, 0x80, 0xa7, 0x90, 0xb5, 0x9e, 0xaa, +}; + +/* + * Contains the SHA-1 thumbprints of certificates that are being revoked by DBX. * This only includes the 'Microsoft Windows Production PCA 2011' for now. */ static uint8_t certdbx[] = { - 0x58, 0x0a, 0x6f, 0x4c, 0xc4, 0xe4, 0xb6, 0x69, 0xb9, 0xeb, 0xdc, 0x1b, 0x2b, 0x3e, 0x08, 0x7b, 0x80, 0xd0, 0x67, 0x8d + 0x58, 0x0a, 0x6f, 0x4c, 0xc4, 0xe4, 0xb6, 0x69, 0xb9, 0xeb, 0xdc, 0x1b, 0x2b, 0x3e, 0x08, 0x7b, 0x80, 0xd0, 0x67, 0x8d, }; /* diff --git a/src/hash.c b/src/hash.c index ee21cc51e1f..88ffca31da5 100644 --- a/src/hash.c +++ b/src/hash.c @@ -2171,37 +2171,51 @@ static BOOL IsRevokedBySvn(uint8_t* buf, uint32_t len) return FALSE; } -static BOOL IsRevokedByCert(uint8_t* buf, uint32_t len) +static BOOL IsRevokedByCert(cert_info_t* info) { int i; - uint8_t* cert; - cert_info_t info; - - cert = GetPeSignatureData(buf); - i = GetIssuerCertificateInfo(cert, &info); - if (i == 0) - uuprintf(" (Unsigned Bootloader)"); - if (i <= 0) - return FALSE; - uuprintf(" Signed by: %s", info.name); for (i = 0; i < ARRAYSIZE(certdbx); i += SHA1_HASHSIZE) { if (!expert_mode) continue; - if (memcmp(info.thumbprint, &certdbx[i], SHA1_HASHSIZE) == 0) { - uprintf("Found '%s' revoked certificate", info.name); + if (memcmp(info->thumbprint, &certdbx[i], SHA1_HASHSIZE) == 0) { + uprintf("Found '%s' revoked certificate", info->name); return TRUE; } } return FALSE; } +BOOL IsSignedBySecureBootAuthority(uint8_t* buf, uint32_t len) +{ + int i; + uint8_t* cert; + cert_info_t info; + + if (buf == NULL || len < 0x100) + return FALSE; + + // Get the signer/issuer info + cert = GetPeSignatureData(buf); + // Secure Boot Authority is always an issuer + if (GetIssuerCertificateInfo(cert, &info) != 2) + return FALSE; + for (i = 0; i < ARRAYSIZE(certauth); i += SHA1_HASHSIZE) { + if (memcmp(info.thumbprint, &certauth[i], SHA1_HASHSIZE) == 0) + return TRUE; + } + return FALSE; +} + int IsBootloaderRevoked(uint8_t* buf, uint32_t len) { uint32_t i; uint8_t hash[SHA256_HASHSIZE]; IMAGE_DOS_HEADER* dos_header = (IMAGE_DOS_HEADER*)buf; IMAGE_NT_HEADERS32* pe_header; + uint8_t* cert; + cert_info_t info; + int r; // Fall back to embedded sbat_level.txt if we couldn't access remote if (sbat_entries == NULL) { @@ -2214,12 +2228,17 @@ int IsBootloaderRevoked(uint8_t* buf, uint32_t len) pe_header = (IMAGE_NT_HEADERS32*)&buf[dos_header->e_lfanew]; if (pe_header->Signature != IMAGE_NT_SIGNATURE) return -2; + + // Get the signer/issuer info + cert = GetPeSignatureData(buf); + r = GetIssuerCertificateInfo(cert, &info); + if (r == 0) + uuprintf(" (Unsigned Bootloader)"); + else if (r > 0) + uuprintf(" Signed by: %s", info.name); + if (!PE256Buffer(buf, len, hash)) return -1; - // Check for UEFI DBX certificate revocation - // This also displays the name of the signer/issuer cert if available - if (IsRevokedByCert(buf, len)) - return 5; // Check for UEFI DBX revocation for (i = 0; i < ARRAYSIZE(pe256dbx); i += SHA256_HASHSIZE) if (memcmp(hash, &pe256dbx[i], SHA256_HASHSIZE) == 0) @@ -2234,6 +2253,9 @@ int IsBootloaderRevoked(uint8_t* buf, uint32_t len) // Check for Microsoft SVN revocation if (IsRevokedBySvn(buf, len)) return 4; + // Check for UEFI DBX certificate revocation + if (IsRevokedByCert(&info)) + return 5; return 0; } diff --git a/src/iso.c b/src/iso.c index ab043607737..45efbbdeb8c 100644 --- a/src/iso.c +++ b/src/iso.c @@ -249,9 +249,10 @@ static BOOL check_iso_props(const char* psz_dirname, int64_t file_length, const // We may extract the bootloaders for revocation validation later but // to do so, since we're working with case sensitive file systems, we // must store all found UEFI bootloader paths with the right case. - for (j = 0; j < ARRAYSIZE(img_report.efi_boot_path); j++) { - if (img_report.efi_boot_path[j][0] == 0) { - static_strcpy(img_report.efi_boot_path[j], psz_fullpath); + for (j = 0; j < ARRAYSIZE(img_report.efi_boot_entry); j++) { + if (img_report.efi_boot_entry[j].path[0] == 0) { + img_report.efi_boot_entry[j].type = EBT_BOOTMGR; + static_strcpy(img_report.efi_boot_entry[j].path, psz_fullpath); break; } } @@ -291,9 +292,10 @@ static BOOL check_iso_props(const char* psz_dirname, int64_t file_length, const if (safe_stricmp(psz_basename, bootloader_name) == 0) { if (k == 0) img_report.has_efi |= (2 << i); // start at 2 since "bootmgr.efi" is bit 0 - for (j = 0; j < ARRAYSIZE(img_report.efi_boot_path); j++) { - if (img_report.efi_boot_path[j][0] == 0) { - static_strcpy(img_report.efi_boot_path[j], psz_fullpath); + for (j = 0; j < ARRAYSIZE(img_report.efi_boot_entry); j++) { + if (img_report.efi_boot_entry[j].path[0] == 0) { + img_report.efi_boot_entry[j].type = (uint8_t)k; + static_strcpy(img_report.efi_boot_entry[j].path, psz_fullpath); break; } } @@ -1559,6 +1561,7 @@ uint32_t ReadISOFileToBuffer(const char* iso, const char* iso_file, uint8_t** bu iso9660_stat_t* p_statbuf = NULL; *buf = NULL; + cdio_loglevel_default = CDIO_LOG_WARN; // First try to open as UDF - fallback to ISO if it failed p_udf = udf_open(iso); @@ -1632,6 +1635,7 @@ uint32_t ReadISOFileToBuffer(const char* iso, const char* iso_file, uint8_t** bu udf_dirent_free(p_udf_file); iso9660_close(p_iso); udf_close(p_udf); + cdio_loglevel_default = usb_debug ? CDIO_LOG_INFO : CDIO_LOG_WARN; if (ret == 0) safe_free(*buf); return ret; diff --git a/src/rufus.c b/src/rufus.c index 0ed225bddfa..b343c0c9251 100755 --- a/src/rufus.c +++ b/src/rufus.c @@ -1609,43 +1609,61 @@ static DWORD WINAPI BootCheckThread(LPVOID param) // Check UEFI bootloaders for revocation if (IS_EFI_BOOTABLE(img_report)) { - assert(ARRAYSIZE(img_report.efi_boot_path) > 0); + BOOL has_secureboot_signed_bootloader = FALSE; + assert(ARRAYSIZE(img_report.efi_boot_entry) > 0); PrintStatus(0, MSG_351); uuprintf("UEFI Secure Boot revocation checks:"); - rr = 0; - for (i = 0; i < ARRAYSIZE(img_report.efi_boot_path) && img_report.efi_boot_path[i][0] != 0; i++) { - static const char* revocation_type[] = { "UEFI DBX", "Windows SSP", "Linux SBAT", "Windows SVN", "Cert DBX" }; - cdio_loglevel_default = CDIO_LOG_WARN; - len = ReadISOFileToBuffer(image_path, img_report.efi_boot_path[i], &buf); - cdio_loglevel_default = usb_debug ? CDIO_LOG_DEBUG : CDIO_LOG_WARN; - if (len == 0) { - uprintf("Warning: Failed to extract '%s' to check for UEFI revocation", img_report.efi_boot_path[i]); - continue; - } - uuprintf("• %s", img_report.efi_boot_path[i]); - r = IsBootloaderRevoked(buf, len); - safe_free(buf); - if (r > 0) { - assert(r <= ARRAYSIZE(revocation_type)); - if (rr == 0) - rr = r; - uprintf("Warning: '%s' has been revoked by %s", img_report.efi_boot_path[i], revocation_type[r - 1]); - is_bootloader_revoked = TRUE; + // Make sure we have at least one regular EFI bootloader that is formally signed + // for Secure Boot, since it doesn't make sense to report revocation otherwise. + for (i = 0; !has_secureboot_signed_bootloader && i < ARRAYSIZE(img_report.efi_boot_entry) && + img_report.efi_boot_entry[i].path[0] != 0; i++) { + if (img_report.efi_boot_entry[i].type == EBT_MAIN) { + len = ReadISOFileToBuffer(image_path, img_report.efi_boot_entry[i].path, &buf); + if (len == 0) { + uprintf("Warning: Failed to extract '%s' to check for UEFI revocation", img_report.efi_boot_entry[i].path); + continue; + } + if (IsSignedBySecureBootAuthority(buf, len)) + has_secureboot_signed_bootloader = TRUE; + free(buf); } } - if (rr > 0) { - switch (rr) { - case 2: - msg = lmprintf(MSG_341, "Error code: 0xc0000428"); - break; - default: - msg = lmprintf(MSG_340); - break; + if (!has_secureboot_signed_bootloader) { + uuprintf(" No Secure Boot signed bootloader found -- skipping"); + } else { + rr = 0; + for (i = 0; i < ARRAYSIZE(img_report.efi_boot_entry) && img_report.efi_boot_entry[i].path[0] != 0; i++) { + static const char* revocation_type[] = { "UEFI DBX", "Windows SSP", "Linux SBAT", "Windows SVN", "Cert DBX" }; + len = ReadISOFileToBuffer(image_path, img_report.efi_boot_entry[i].path, &buf); + if (len == 0) { + uprintf("Warning: Failed to extract '%s' to check for UEFI revocation", img_report.efi_boot_entry[i].path); + continue; + } + uuprintf("• %s", img_report.efi_boot_entry[i].path); + r = IsBootloaderRevoked(buf, len); + safe_free(buf); + if (r > 0) { + assert(r <= ARRAYSIZE(revocation_type)); + if (rr == 0) + rr = r; + uprintf("Warning: '%s' has been revoked by %s", img_report.efi_boot_entry[i].path, revocation_type[r - 1]); + is_bootloader_revoked = TRUE; + } + } + if (rr > 0) { + switch (rr) { + case 2: + msg = lmprintf(MSG_341, "Error code: 0xc0000428"); + break; + default: + msg = lmprintf(MSG_340); + break; + } + r = MessageBoxExU(hMainDialog, lmprintf(MSG_339, msg), lmprintf(MSG_338), + MB_OKCANCEL | MB_ICONWARNING | MB_IS_RTL, selected_langid); + if (r == IDCANCEL) + goto out; } - r = MessageBoxExU(hMainDialog, lmprintf(MSG_339, msg), lmprintf(MSG_338), - MB_OKCANCEL | MB_ICONWARNING | MB_IS_RTL, selected_langid); - if (r == IDCANCEL) - goto out; } } @@ -3516,7 +3534,7 @@ int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine is_vds_available = IsVdsAvailable(FALSE); use_vds = ReadSettingBool(SETTING_USE_VDS) && is_vds_available; usb_debug = ReadSettingBool(SETTING_ENABLE_USB_DEBUG); - cdio_loglevel_default = usb_debug ? CDIO_LOG_DEBUG : CDIO_LOG_WARN; + cdio_loglevel_default = usb_debug ? CDIO_LOG_INFO : CDIO_LOG_WARN; use_rufus_mbr = !ReadSettingBool(SETTING_DISABLE_RUFUS_MBR); // validate_md5sum = ReadSettingBool(SETTING_ENABLE_RUNTIME_VALIDATION); detect_fakes = !ReadSettingBool(SETTING_DISABLE_FAKE_DRIVES_CHECK); @@ -3807,7 +3825,7 @@ extern int TestHashes(void); // Alt-. => Enable USB enumeration debug if ((msg.message == WM_SYSKEYDOWN) && (msg.wParam == VK_OEM_PERIOD)) { usb_debug = !usb_debug; - cdio_loglevel_default = usb_debug ? CDIO_LOG_DEBUG : CDIO_LOG_WARN; + cdio_loglevel_default = usb_debug ? CDIO_LOG_INFO : CDIO_LOG_WARN; WriteSettingBool(SETTING_ENABLE_USB_DEBUG, usb_debug); PrintStatusTimeout(lmprintf(MSG_270), usb_debug); GetDevices(0); diff --git a/src/rufus.h b/src/rufus.h index 0f3d44bc911..540a1bd7682 100644 --- a/src/rufus.h +++ b/src/rufus.h @@ -327,6 +327,13 @@ enum file_io_type { FILE_IO_APPEND }; +enum EFI_BOOT_TYPE { + EBT_MAIN = 0, + EBT_GRUB, + EBT_MOKMANAGER, + EBT_BOOTMGR +}; + /* Special handling for old .c32 files we need to replace */ #define NB_OLD_C32 2 #define OLD_C32_NAMES { "menu.c32", "vesamenu.c32" } @@ -383,13 +390,18 @@ enum ArchType { ARCH_MAX }; +typedef struct { + uint8_t type; + char path[64]; +} efi_boot_entry_t; + typedef struct { char label[192]; // 3*64 to account for UTF-8 char usb_label[192]; // converted USB label for workaround char cfg_path[128]; // path to the ISO's isolinux.cfg char reactos_path[128]; // path to the ISO's freeldr.sys or setupldr.sys char wininst_path[MAX_WININST][64]; // path to the Windows install image(s) - char efi_boot_path[64][32]; // paths of detected UEFI bootloaders + efi_boot_entry_t efi_boot_entry[64];// types and paths of detected UEFI bootloaders char efi_img_path[128]; // path to an efi.img file uint64_t image_size; uint64_t projected_size; @@ -823,6 +835,7 @@ extern BOOL PE256Buffer(uint8_t* buf, uint32_t len, uint8_t* hash); extern void UpdateMD5Sum(const char* dest_dir, const char* md5sum_name); extern BOOL HashBuffer(const unsigned type, const uint8_t* buf, const size_t len, uint8_t* sum); extern BOOL IsFileInDB(const char* path); +extern BOOL IsSignedBySecureBootAuthority(uint8_t* buf, uint32_t len); extern int IsBootloaderRevoked(uint8_t* buf, uint32_t len); extern void PrintRevokedBootloaderInfo(void); extern BOOL IsBufferInDB(const unsigned char* buf, const size_t len); diff --git a/src/rufus.rc b/src/rufus.rc index cffac279efd..2e094160c15 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2204" +CAPTION "Rufus 4.6.2205" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -399,8 +399,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2204,0 - PRODUCTVERSION 4,6,2204,0 + FILEVERSION 4,6,2205,0 + PRODUCTVERSION 4,6,2205,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -418,13 +418,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2204" + VALUE "FileVersion", "4.6.2205" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", " 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2204" + VALUE "ProductVersion", "4.6.2205" END END BLOCK "VarFileInfo" From eb282642ff8981d663f9e5807d7d6a94fe3d708e Mon Sep 17 00:00:00 2001 From: Pete Batard <pete@akeo.ie> Date: Mon, 14 Oct 2024 17:39:17 +0100 Subject: [PATCH 24/30] [misc] fix conversion of windows error codes * Commit f453dc272b93adb0d63dd045197f4661fc653211 improved on error message reporting, but went a bit too far in trying to let Windows facilities sort their messages out. * Add a retry that clears the facility, so that, for one thing, we get wininet messages properly processed, regardless of the official facility assigned. --- src/rufus.rc | 10 +++++----- src/stdio.c | 20 +++++++++++++------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/rufus.rc b/src/rufus.rc index 2e094160c15..7d3c20aee04 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2205" +CAPTION "Rufus 4.6.2206" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -399,8 +399,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2205,0 - PRODUCTVERSION 4,6,2205,0 + FILEVERSION 4,6,2206,0 + PRODUCTVERSION 4,6,2206,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -418,13 +418,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2205" + VALUE "FileVersion", "4.6.2206" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", " 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2205" + VALUE "ProductVersion", "4.6.2206" END END BLOCK "VarFileInfo" diff --git a/src/stdio.c b/src/stdio.c index a6c257b403e..db6d9086ed3 100644 --- a/src/stdio.c +++ b/src/stdio.c @@ -233,16 +233,18 @@ const char *WindowsErrorString(void) static char err_string[256] = { 0 }; DWORD size, presize; - DWORD error_code, format_error; + DWORD error_code, _error_code, format_error; HANDLE hModule = NULL; error_code = GetLastError(); + _error_code = error_code; +retry: // Check for specific facility error codes - switch (HRESULT_FACILITY(error_code)) { + switch (HRESULT_FACILITY(_error_code)) { case FACILITY_NULL: // Special case for internet related errors, that don't actually have a facility // set but still require a hModule into wininet to display the messages. - if ((error_code >= INTERNET_ERROR_BASE) && (error_code <= INTERNET_ERROR_LAST)) + if ((_error_code >= INTERNET_ERROR_BASE) && (_error_code <= INTERNET_ERROR_LAST)) hModule = GetModuleHandleA("wininet.dll"); break; case FACILITY_ITF: @@ -260,22 +262,26 @@ const char *WindowsErrorString(void) // coverity[var_deref_model] size = FormatMessageU(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS | ((hModule != NULL) ? FORMAT_MESSAGE_FROM_HMODULE : 0), hModule, - error_code, MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US), + _error_code, MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US), &err_string[presize], (DWORD)(sizeof(err_string) - strlen(err_string)), NULL); if (size == 0) { format_error = GetLastError(); switch (format_error) { case ERROR_SUCCESS: - static_sprintf(err_string, "[0x%08lX] (No Windows Error String)", error_code); + static_sprintf(err_string, "[0x%08lX] (No Windows Error String)", _error_code); break; case ERROR_MR_MID_NOT_FOUND: case ERROR_MUI_FILE_NOT_FOUND: case ERROR_MUI_FILE_NOT_LOADED: + // We might be trying with the wrong facility. Remove it and try again. + if (HRESULT_FACILITY(_error_code) != FACILITY_NULL) { + _error_code = HRESULT_CODE(_error_code); + goto retry; + } static_sprintf(err_string, "[0x%08lX] (NB: This system was unable to provide an English error message)", error_code); break; default: - static_sprintf(err_string, "[0x%08lX] (FormatMessage error code 0x%08lX)", - error_code, format_error); + static_sprintf(err_string, "[0x%08lX] (FormatMessage error code 0x%08lX)", error_code, format_error); break; } } else { From accc7c000bf833fb2606002a80cced5774f0907c Mon Sep 17 00:00:00 2001 From: Justin Miller <justin.miller@reactos.org> Date: Thu, 17 Oct 2024 14:47:52 -0700 Subject: [PATCH 25/30] [iso] check for freeldr.sys as well for ReactOS * Closes #2589. Signed-off-by: Justin Miller <justin.miller@reactos.org> --- src/iso.c | 11 +++++++---- src/rufus.rc | 10 +++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/iso.c b/src/iso.c index 45efbbdeb8c..664136569ba 100644 --- a/src/iso.c +++ b/src/iso.c @@ -112,7 +112,7 @@ static const char* syslinux_cfg[] = { "isolinux.cfg", "syslinux.cfg", "extlinux. static const char* isolinux_bin[] = { "isolinux.bin", "boot.bin" }; static const char* pe_dirname[] = { "/i386", "/amd64", "/minint" }; static const char* pe_file[] = { "ntdetect.com", "setupldr.bin", "txtsetup.sif" }; -static const char* reactos_name = "setupldr.sys"; // TODO: freeldr.sys doesn't seem to work +static const char* reactos_name[] = { "setupldr.sys", "freeldr.sys" }; static const char* kolibri_name = "kolibri.img"; static const char* autorun_name = "autorun.inf"; static const char* manjaro_marker = ".miso"; @@ -274,9 +274,12 @@ static BOOL check_iso_props(const char* psz_dirname, int64_t file_length, const } } - // Check for ReactOS' setupldr.sys anywhere - if ((img_report.reactos_path[0] == 0) && (safe_stricmp(psz_basename, reactos_name) == 0)) - static_strcpy(img_report.reactos_path, psz_fullpath); + // Check for ReactOS presence anywhere + if (img_report.reactos_path[0] == 0) { + for (i = 0; i < ARRAYSIZE(reactos_name); i++) + if (safe_stricmp(psz_basename, reactos_name[i]) == 0) + static_strcpy(img_report.reactos_path, psz_fullpath); + } // Check for the first 'efi*.img' we can find (that hopefully contains EFI boot files) if (!HAS_EFI_IMG(img_report) && (safe_strlen(psz_basename) >= 7) && diff --git a/src/rufus.rc b/src/rufus.rc index 7d3c20aee04..bf9da2325c7 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2206" +CAPTION "Rufus 4.6.2207" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -399,8 +399,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2206,0 - PRODUCTVERSION 4,6,2206,0 + FILEVERSION 4,6,2207,0 + PRODUCTVERSION 4,6,2207,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -418,13 +418,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2206" + VALUE "FileVersion", "4.6.2207" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", " 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2206" + VALUE "ProductVersion", "4.6.2207" END END BLOCK "VarFileInfo" From 7488e4464d9c46904fd0fff1d1128ad0fc50f2b3 Mon Sep 17 00:00:00 2001 From: Pete Batard <pete@akeo.ie> Date: Mon, 21 Oct 2024 15:38:40 +0100 Subject: [PATCH 26/30] Rufus 4.6 (Build 2208) --- ChangeLog.txt | 3 ++- res/appstore/listing/listing.csv | 43 ++++++++++++++++---------------- res/loc/ChangeLog.txt | 4 +++ src/rufus.rc | 10 ++++---- 4 files changed, 32 insertions(+), 28 deletions(-) diff --git a/ChangeLog.txt b/ChangeLog.txt index 66f61081454..b19101d0086 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -1,10 +1,11 @@ -o Version 4.6 (2024.10.??) +o Version 4.6 (2024.10.21) Add a new setup.exe wrapper to bypass Windows 11 24H2 in-place upgrade restrictions Add TimeZone to regional options replication Set local account passwords to not expire by default Fix an error when trying to write compressed VHD images Fix an error when invoking Rufus from the PowerShell commandline Improve revoked UEFI bootloaders check to support Linux SBAT, Windows SVN and cert DBX + Improve support for ReactOS boot media o Version 4.5 (2024.05.22) Add new advanced option to perform runtime UEFI media validation of suitable images (Windows, most Linux) diff --git a/res/appstore/listing/listing.csv b/res/appstore/listing/listing.csv index 6df2d8c7143..4a5bf2da70d 100644 --- a/res/appstore/listing/listing.csv +++ b/res/appstore/listing/listing.csv @@ -1,4 +1,4 @@ -"Field","ID","Type (Type)","default","en-us","ar-sa","bg-bg","zh-cn","zh-tw","hr-hr","cs-cz","da-dk","nl-nl","fi-fi","fr-fr","de-de","el-gr","he-il","hu-hu","id-id","it-it","ja-jp","ko-kr","lv-lv","lt-lt","ms-my","nb-no","fa-ir","pl-pl","pt-br","pt-pt","ro-ro","ru-ru","sr-latn-rs","sk-sk","sl-si","es-es","sv-se","th-th","tr-tr","uk-ua","vi-vn" +"Field","ID","Type (Type)","default","en-us","ar-sa","bg-bg","zh-cn","zh-tw","hr-hr","cs-cz","da-dk","nl-nl","fi-fi","fr-fr","de-de","el-gr","he-il","hu-hu","id-id","it-it","ja-jp","ko-kr","lv-lv","lt-lt","ms-my","nb-no","fa-ir","pl-pl","pt-br","pt-pt","ro-ro","ru-ru","sr-latn-rs","sk-sk","sl-si","es-es","sv-se","th-th","tr-tr","uk-ua","vi-vn" "Description","2","Text","","Rufus is a utility that helps format and create bootable USB flash drives, such as USB keys/pendrives, memory sticks, etc. • Official site: https://rufus.ie • Source Code: https://github.com/pbatard/rufus @@ -11,9 +11,9 @@ • Лог на промените: https://github.com/pbatard/rufus/blob/master/ChangeLog.txt","Rufus 是一个帮助您格式化并创建可启动 USB 驱动器（如 U 盘和存储卡）的工具。 • 官方网站: https://rufus.ie • 源代码: https://github.com/pbatard/rufus -• 更新日志: https://github.com/pbatard/rufus/blob/master/ChangeLog.txt","Rufus 是個能格式化並製作可開機 USB 快閃磁碟機（USB 隨身碟、Memory Stick 等等）的工具。 +• 更新日志: https://github.com/pbatard/rufus/blob/master/ChangeLog.txt","Rufus 是個能格式化並製作可開機 USB 快閃磁碟 (如 USB 隨身碟等) 的工具。 • 官方網站: https://rufus.ie -• 源始碼: https://github.com/pbatard/rufus +• 原始碼: https://github.com/pbatard/rufus • 更新日誌: https://github.com/pbatard/rufus/blob/master/ChangeLog.txt","Rufus je aplikacija koja olakšava formatiranje i stvaranje USB pokretačkih jedinica. • Službena stranica: https://rufus.ie • Šifra izvora: https://github.com/pbatard/rufus @@ -71,7 +71,7 @@ • Endringslogg: https://github.com/pbatard/rufus/blob/master/ChangeLog.txt","Rufus ابزاری است که به فرمت و ایجاد درایوهای فلش USB قابل بوت، مانند کلیدهای USB/pendrives، مموری استیک ها و غیره به شما کمک می کند. • سایت رسمی: https://rufus.ie • منبع کد: https://github.com/pbatard/rufus -• لیست تغییرات https://github.com/pbatard/rufus/blob/master/ChangeLog.txt","Rufus to narzędzie, które pomaga formatować i tworzyć bootowalne dyski flash USB, takie jak klucze USB / pendrive, pendrive'y itp. +• لیست تغییرات https://github.com/pbatard/rufus/blob/master/ChangeLog.txt","Rufus to narzędzie, które pomaga formatować i tworzyć dyski rozruchowe flash USB, takie jak klucze USB, pendrive-y itp. • Oficjalna strona: https://rufus.ie • Kod źródłowy: https://github.com/pbatard/rufus • Zmiany: https://github.com/pbatard/rufus/blob/master/ChangeLog.txt","Rufus é um utilitário que ajuda a formatar e a criar unidades flash USB inicializáveis, tais como pendrives USB, cartões de memória, etc. @@ -114,16 +114,14 @@ • Trang web chính thức: https://rufus.ie • Mã nguồn: https://github.com/pbatard/rufus • Nhật ký thay đổi: https://github.com/pbatard/rufus/blob/master/ChangeLog.txt" -"ReleaseNotes","3","Text","• Add new advanced option to perform runtime UEFI media validation of suitable images (Windows, most Linux) -• Move the 'Use Rufus MBR' advanced option to a cheat mode (Alt-A) -• Fix truncation of VHDX images, as well as a benign error message when writing VHD/VHDX -• Fix support for Linux persistence in some configurations (Mint, Ubuntu 24.04) -• Fix multiple potential vulnerabilities (with thanks to Mansour Gashasbi) -• Update internal GRUB to version 2.12 -• Update UEFI:NTFS to latest (now always uses the ntfs-3g driver, rather than the buggy AMI NTFS one) -• Increase buffer size when copying ISO files, in an attempt to minimize the AMI NTFS UEFI driver bug -• Improve partition creation handling -• Don't display the WUE dialog when a conflicting 'unattend.xml' already exists",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +"ReleaseNotes","3","Text","• Add a new setup.exe wrapper to bypass Windows 11 24H2 in-place upgrade restrictions +• Add TimeZone to regional options replication +• Set local account passwords to not expire by default +• Fix an error when trying to write compressed VHD images +• Fix an error when invoking Rufus from the PowerShell commandline +• Improve revoked UEFI bootloaders check to support Linux SBAT, Windows SVN and cert DBX +• Improve support for ReactOS boot media + ",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, "Title","4","Text","Rufus",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, "ShortTitle","5","Text","",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, "SortTitle","6","Text","",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, @@ -265,16 +263,17 @@ Xem tại https://www.gnu.org/licenses/gpl-3.0.html để biết thêm chi tiế "OptionalPromo1000x800","612","Relative path (or URL to file in Partner Center)","",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, "OptionalPromo414x180","613","Relative path (or URL to file in Partner Center)","",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, "Feature1","700","Text","","Format USB, flash card and virtual drives to FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","تهيئة USB وبطاقة الفلاش ومحركات الأقراص الافتراضية إلى FAT / FAT32 / NTFS / UDF / exFAT / ReFS / ext2 / ext3","Форматиране на USB, карти памет и виртуални устройства с FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","将 U 盘、存储卡或虚拟驱动器格式化为 FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3 格式","將隨身碟、記憶卡或虛擬光碟機格式化為 FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3 格式","Formatirajte USB, flash karticu i virtualne pogone na FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Formátování USB, flash karet a virtuálních jednotek na FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Formater USB, flash kort og virtuelle drev til FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","USB, flashkaart en virtuele schijven formatteren naar FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Alusta USB-asemia, muistikortteja ja virtuaalisia asemia muotoon FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Formatez des périphériques USB, des cartes flash et des disques virtuels en FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Formatieren von USB, Flash-Karte und virtuellen Laufwerken in FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Μορφοποίηση USB, κάρτας flash και εικονικών μονάδων δίσκου σε FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","אתחול USB, כרטיסי זיכרון וכוננים וירטואליים ל־FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","USB, flash memóriakártyák és virtuális meghajtók formázása FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3 fájlrendszerre","Format USB, kartu flash dan penyimpanan maya ke FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Formatta USB, flash card e unità virtuali in FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","USBメモリやSDカード、仮想ドライブをFAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3でフォーマットします。","USB, 플래시 카드 및 가상 드라이브를 FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3로 포맷","Formatē USB, atmiņas kartes un virtuālos diskus formātos FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Suformatuokite USB, flash kortelę ir virtualius diskus į FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Formatkan USB, kad flash dan pemacu maya kepada FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Formater USB, minnekort og virtuelle disker til FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","فرمت USB، فلش کارت و درایوهای مجازی به FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Sformatuj nośnik używając: FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Formatar dispositivos USB, cartões flash e discos virtuais com FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Formatar dispositivos USB, cartão de memória e drives virtuais em FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Formatează USB, card flash si drive-uri virtuale la FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Форматирование USB, флешек и виртуальных дисков в FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Formatiraj USB, flash kartice, i virtualne diskove u FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Naformátujte USB, kartu a virtuálne disky do FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Formatiranje USB, bliskavice in virtualnih pogonov na FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Formatee USB, tarjetas flash y unidades virtuales a FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Formatera USB-enheter, flash-kort och virtuella enheter till FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","ฟอร์แมต USB, แฟลชการ์ด หรือไดรฟ์เสมือน ให้อยู่ในรูปแบบของ FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","USB, flash kart ve sanal sürücüleri FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3 olarak biçimlendirin","Форматування USB-накопичувачів, флешок, карток пам'яті у FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3","Định dạng USB, thẻ nhớ hoặc ổ nhớ ảo với FAT/FAT32/NTFS/UDF/exFAT/ReFS/ext2/ext3" -"Feature2","701","Text","","Create FreeDOS bootable USB drives","إنشاء محركات أقراص USB قابلة للإقلاع من FreeDOS","Създаване на FreeDOS стартиращ USB устройства","创建 FreeDOS 可启动驱动器","建立 FreeDOS 可啟隨身碟","Stvaranje FreeDOS USB pogona za pokretanje","Vytvoření bootovacích disků USB se systémem FreeDOS","Lav FreeDOS opstartsbare USB drev","FreeDOS opstartbare USB-schijven aanmaken","Luo boottaavia FreeDOS USB-asemia","Créez des disques amorçable FreeDOS","FreeDOS-bootfähige USB-Laufwerke erstellen","Δημιουργήστε μονάδες USB με δυνατότητα εκκίνησης FreeDOS","יצירת כונני USB הניתנים לאתחול של FreeDOS","Bootolható FreeDOS USB meghajtó készítése","Buat perangkat USB FreeDOS yang dapat di boot","Crea unità USB avviabili FreeDOS","FreeDOSの起動可能ドライブを作成します。","FreeDOS 부팅 가능한 USB 드라이브 만들기","Izveido FreeDOS ielādes USB ierīces","Sukurkite FreeDOS įkrovos USB diskus","Buat pemacu USB boleh boot FreeDOS","Lag FreeDos oppstartbar USB stick","درایوهای USB قابل بوت FreeDOS را ایجاد کنید","Stwórz bootowalny nośnik USB FreeDOS","Criar discos USB inicializáveis FreeDOS","Criar unidades USB inicializáveis FreeDOS","Crează drive USB bootabil FreeDOS","Создание загрузочных USB-дисков FreeDOS","Kreiraj FreeDOS butabilni USB disk","Vytvorte bootovacie usb zariadenia FreeDOS","Ustvarite FreeDOS zagonske USB pogone","Crear unidades USB de arranque FreeDOS","Skapa FreeDOS startbara USB-enheter","สร้าง USB ไดรฟ์บูตสำหรับระบบ FreeDOS","FreeDOS önyüklenebilir USB sürücüleri oluşturun","Створення завантажувальних пристроїв FreeDOS","Tạo USB có thể khởi động với FreeDOS" -"Feature3","702","Text","","Create bootable drives from bootable ISOs (Windows, Linux, etc.)","إنشاء محركات أقراص قابلة للإقلاع من ملفات ISO القابلة للإقلاع (Windows و Linux وما إلى ذلك)","Създаване на стартиращи устройства от ISO образи (Windows, Linux и др.)","从可启动 ISO 文件 (Windows 和 Linux 等) 创建可启动驱动器","從可啟動 ISO 檔案 (Windows 和 Linux 等) 建立可啟動隨身碟","Stvaranje pogona za pokretanje iz ISO-ova za pokretanje (Windows, Linux itd.)","Vytváření bootovacích jednotek ze zaváděcích ISO (Windows, Linux atd.)","Lav opstartsbarer drev fra opstartsbarer ISOer (Window, Linux, osv.)","Opstartbare schijven aanmaken via opstartbare ISO's (Windows, Linux, enz)","Luo käynnistysasemia boottaavista ISO-kuvista (Windows, Linux jne.)","Créez des disques amorçables à partir d'images ISOs (Windows, Linux, etc.)","Erstellen bootfähiger Laufwerke aus bootfähigen ISOs (Windows, Linux, etc.)","Δημιουργήστε εκκινήσιμες μονάδες από ISO με δυνατότητα εκκίνησης (Windows, Linux, κ.λπ.)","יצירת כוננים הניתנים לאתחול מקובצי ISO הניתנים לאתחול (Windows‏, Linux וכו')","Bootolható meghajtók készítése bootolható ISO képfájlokból (Windows, Linux, stb.)","Buat perangkat yang dapat di boot dari ISO (Windows, Linux, dll.)","Crea unità avviabili da ISO avviabili (Windows, Linux, ecc.)","WindowsやLinuxなどのISOファイルから起動可能ドライブを作成します。","부팅 가능한 ISO (Windows, Linux 등)에서 부팅 가능한 드라이브 만들기","Izveido ielādes ierīces no ISO failiem (Windows, Linux, u.c.)","Sukurkite įkrovos diskus iš įkrovos ISO (Windows, Linux ir kt.)","Buat pemacu boleh boot daripada ISO boleh boot (Windows, Linux, dll.)","Lag oppstartabar enhet/disk fra ISOer (Windows, Linux, etc.)","ایجاد درایوهای قابل بوت از ISOهای قابل بوت (ویندوز، لینوکس و غیره)","Twórz dyski rozruchowe z obrazów ISO (Windows, Linux itp.)","Criar discos inicializáveis a partir de ISOs inicializáveis (Windows, Linux, etc.)","Criar unidades inicializáveis a partir de ISOs inicializáveis (Windows, Linux, etc.)","Crează drive-uri bootabile de la ISO-uri bootabile (Windows, Linux, etc.)","Создание загрузочных дисков из загрузочных образов ISO (Windows, Linux и т.д.)","Kreirajte disk jedinice za pokretanje sistema od ISO-a koji se mogu pokretanja sistema (Windows, Linux itd.)","Vytvorte bootovacie jednotky z ISO súborov (Windows, Linux atď.)","Ustvarite zagonske pogone iz zagonskih ISO-jev (Windows, Linux itd.)","Cree unidades de arranque desde ISO de arranque (Windows, Linux, etc.)","Skapa startbara enheter från startbara ISO-filer (Windows, Linux, etc.)","สร้าง USB ไดรฟ์บูตจากไฟล์ ISO ที่บูตได้ (เช่น ไฟล์ติดตั้ง Windows หรือ Linux เป็นต้น)","Önyüklenebilir ISO'lardan önyüklenebilir sürücüler oluşturun (Windows, Linux, vb.)","Створення завантажувальних пристроїв з завантажувальних образів (Windows, Linux, тощо)","Tạo ổ đĩa có thể khởi động từ các file ISO có thể khởi động (Windows, Linux, v.v...)" -"Feature4","703","Text","","Create bootable drives from bootable disk images, including compressed ones","إنشاء محركات أقراص قابلة للإقلاع من صور الأقراص القابلة للإقلاع ، بما في ذلك الأقراص المضغوطة","Създаване на стартиращи устройства от образи, включително компресирани такива","从可启动硬盘镜像 (包括压缩镜像) 创建可启动驱动器","從可啟動硬碟映像檔 (包括壓縮映像檔) 建立可啟動隨身碟","Stvaranje pogona za pokretanje iz slika diska za pokretanje, uključujući komprimirane","Vytváření zaváděcích jednotek ze zaváděcích obrazů disků, včetně komprimovaných","Lav opstartsbarer drev fra opstartsbarer disk billeder, inklusiv komprimerede billeder","Opstartbare schijven aanmaken van opstartbare schijf-images, inclusief gecomprimeerde images","Luo käynnistysasemia boottaavista levykuvista, pakatut kuvat mukaanlukien","Créez des disques amorçables à partir d'images disque, y compris à partir d'images compressées","Erstellen bootfähiger Laufwerke aus bootfähigen Festplatten-Images, einschließlich komprimierter Images","Δημιουργήστε μονάδες εκκίνησης από εικόνες δίσκου με δυνατότητα εκκίνησης, συμπεριλαμβανομένων συμπιεσμένων","יצירת כוננים הניתנים לאתחול מקובצי תמונת דיסק הניתנים לאתחול, כולל קבצים דחוסים","Bootolható meghajtók készítése bootolható lemez képfájlokból, beleértve a tömörítetteket is","Buat perangkat yang dapat di boot dari Disk Image, termasuk yang Disk Image yang terkompresi","Crea unità avviabili da immagini disco avviabili, incluse quelle compresse","圧縮済みのものを含むディスクイメージから起動可能ドライブを作成します。","압축된 이미지를 포함하여 부팅 가능한 디스크 이미지에서 부팅 가능한 드라이브 만들기","Izveido ielādes ierīces no ielādes disku virtuālajiem attēliem, tai skaitā arī no saspiestajiem","Sukurkite įkrovos diskus iš įkrovos disko vaizdų, įskaitant suspaustus","Buat pemacu boleh boot daripada imej cakera boleh boot, termasuk yang dimampatkan","Lag oppstartbare disker fra images, inkludert komprimerte sådan","درایوهای قابل بوت را از تصاویر دیسک قابل بوت، از جمله موارد فشرده، ایجاد کنید","Twórz dyski rozruchowe z obrazów dysków, włączając skompresowane","Criar discos inicializáveis a partir de imagens de disco inicializáveis, inclusive de imagens compactadas","Criar unidades inicializáveis a partir de imagens de disco inicializáveis, inclusive de imagens compactadas","Crează drive-uri de la imagini de disc bootabile, incluzând cele compresate","Создание загрузочных дисков из образов загрузочных дисков, в том числе сжатых","Kreiranje disk jedinica za pokretanje sistema sa slika diska koji se može pokretanja, uključujući kompresovane","Vytvorte bootovacie jednotky z diskových obrazov, vrátane tých komprimovaných","Ustvarite zagonske pogone iz slik diska, ki jih je mogoče zagnati, vključno s stisnjenimi","Cree unidades de arranque a partir de imágenes de disco de arranque, incluidas las comprimidas","Skapa startbara enheter från startbara diskavbildningar, inklusive komprimerade","สร้างไดรฟ์บูตจากดิสก์อิมเมจที่บูตได้ (รองรับไฟล์ดิสก์อิมเมจที่ถูกบีบอัด)","Sıkıştırılmış olanlar da dahil olmak üzere önyüklenebilir disk yansısından önyüklenebilir sürücüler oluşturun","Створення завантажувальних пристроїв з завантажувальних образів, у тому числі стиснутих","Tạo ổ đĩa có thể khởi động từ các tệp đĩa có thể khởi động, bao gồm cả tệp nén" -"Feature5","704","Text","","Create BIOS or UEFI bootable drives, including UEFI bootable NTFS","إنشاء BIOS أو محركات أقراص UEFI قابلة للإقلاع ، بما في ذلك UEFI NTFS القابل للتشغيل","Създаване на BIOS или UEFI стартиращи устройства, включително UEFI стартиращ NTFS","创建 BIOS 或 UEFI 可启动驱动器，包括 UEFI 可启动的 NTFS 驱动器","建立 BIOS 或 UEFI 可啟動隨身碟，包括 UEFI 可啟動的 NTFS 隨身碟","Stvaranje BIOS ili UEFI pogona za pokretanje, uključujući UEFI NTFS za pokretanje","Vytvořte zaváděcí jednotky BIOS nebo UEFI, včetně zaváděcích souborů NTFS UEFI","Lav BIOS eller UEFI opstartsbarer drev, inklusiv UEFI opstartsbarer NTFS","BIOS of UEFI opstartbare schijven aanmaken, inclusief UEFI opstartbare NTFS","Luo BIOS- tai UEFI-boottaavia asemia, mukaanlukien UEFI-boottaavat NTFS-asemat","Créez des disques amorçables BIOS ou UEFI, y compris des disques UEFI amorçables utilisant NTFS","Erstellen von BIOS- oder UEFI-bootfähigen Laufwerken, einschließlich UEFI-bootfähigem NTFS","Δημιουργήστε μονάδες δίσκου με δυνατότητα εκκίνησης BIOS ή UEFI, συμπεριλαμβανομένων NTFS με δυνατότητα εκκίνησης UEFI","יצירת כוננים הניתנים לאתחול ממחשבים התומכים ב־BIOS או UEFI, לרבות כוננים הניתנים לאתחול מ־UEFI, המשתמשים במערכת הקבצים NTFS","BIOS-ból vagy UEFI-ből bootolható meghajtók készítése, beleértve az UEFI-ből bootolható NTFS meghajtókat is","Buat perangkat BIOS atau UEFI yang dapat di boot, termasuk perangkat NTFS yang dapat di boot oleh UEFI","Crea unità avviabili BIOS o UEFI, incluso NTFS avviabile UEFI","UEFI:NTFSを含むBIOS及びUEFIで起動可能なドライブを作成します。","UEFI 부팅 가능한 NTFS를 포함하여 BIOS 또는 UEFI 부팅 가능 드라이브 만들기","Izveido BIOS vai UEFI ielādes ierīces, ieskaitot UEFI ielādi no NTFS","Sukurkite BIOS arba UEFI įkrovos diskus, įskaitant UEFI įkrovos NTFS","Buat pemacu boleh boot BIOS atau UEFI, termasuk NTFS boleh boot UEFI","Lag BIOS eller UEFI oppstartbare disker, inkludert UEFI oppstartbar NTFS","درایوهای قابل بوت بایوس یا UEFI از جمله NTFS قابل بوت UEFI ایجاد کنید","Stwórz dysk rozruchowy BIOS lub UEFI, włączając bootowalny dysk UEFI NTFS","Criar discos inicializáveis ​​BIOS ou UEFI, inclusive discos UEFI inicializáveis ​​usando NTFS","Criar discos inicializáveis ​​BIOS ou UEFI, inclusive discos UEFI inicializáveis ​​usando NTFS","Crează discuri bootabile BIOS sau UEFI, incluzând NTFS-uri bootabile de UEFI","Создание загрузочных дисков BIOS или UEFI, включая загрузочный UEFI NTFS","Kreiranje BIOS ili UEFI disk jedinica za pokretanje sistema, uključujući NTFS sa UEFI pokretanjem sistema","Vytvorte bootovacie jednotky systému BIOS alebo UEFI vrátane UEFI bootovateľnej jednotky NTFS","Ustvarite zagonske pogone BIOS ali UEFI, vključno z UEFI bootable NTFS","Cree unidades de arranque BIOS o UEFI, incluido NTFS de arranque UEFI","Skapa BIOS- eller UEFI-startbara enheter, inklusive UEFI-startbar NTFS","สร้างไดรฟ์บูตสำหรับระบบ BIOS หรือ UEFI และ UEFI bootable NTFS","UEFI önyüklenebilir NTFS dahil BIOS ya da UEFI önyüklenebilir sürücüler oluşturun","Створення завантажувальних пристроїв BIOS чи UEFI, включаючи завантажувальний UEFI NTFS","Tạo ổ đĩa có thể khởi động với hệ thống BIOS hoặc UEFI, bao gồm cả NTFS có thể khởi động với UEFI" +"Feature2","701","Text","","Create FreeDOS bootable USB drives","إنشاء محركات أقراص USB قابلة للإقلاع من FreeDOS","Създаване на FreeDOS стартиращ USB устройства","创建 FreeDOS 可启动驱动器","建立 FreeDOS 可開機隨身碟","Stvaranje FreeDOS USB pogona za pokretanje","Vytvoření bootovacích disků USB se systémem FreeDOS","Lav FreeDOS opstartsbare USB drev","FreeDOS opstartbare USB-schijven aanmaken","Luo boottaavia FreeDOS USB-asemia","Créez des disques amorçable FreeDOS","FreeDOS-bootfähige USB-Laufwerke erstellen","Δημιουργήστε μονάδες USB με δυνατότητα εκκίνησης FreeDOS","יצירת כונני USB הניתנים לאתחול של FreeDOS","Bootolható FreeDOS USB meghajtó készítése","Buat perangkat USB FreeDOS yang dapat di boot","Crea unità USB avviabili FreeDOS","FreeDOSの起動可能ドライブを作成します。","FreeDOS 부팅 가능한 USB 드라이브 만들기","Izveido FreeDOS ielādes USB ierīces","Sukurkite FreeDOS įkrovos USB diskus","Buat pemacu USB boleh boot FreeDOS","Lag FreeDos oppstartbar USB stick","درایوهای USB قابل بوت FreeDOS را ایجاد کنید","Stwórz nośnik rozruchowy USB FreeDOS","Criar discos USB inicializáveis FreeDOS","Criar unidades USB inicializáveis FreeDOS","Crează drive USB bootabil FreeDOS","Создание загрузочных USB-дисков FreeDOS","Kreiraj FreeDOS butabilni USB disk","Vytvorte bootovacie usb zariadenia FreeDOS","Ustvarite FreeDOS zagonske USB pogone","Crear unidades USB de arranque FreeDOS","Skapa FreeDOS startbara USB-enheter","สร้าง USB ไดรฟ์บูตสำหรับระบบ FreeDOS","FreeDOS önyüklenebilir USB sürücüleri oluşturun","Створення завантажувальних пристроїв FreeDOS","Tạo USB có thể khởi động với FreeDOS" +"Feature3","702","Text","","Create bootable drives from bootable ISOs (Windows, Linux, etc.)","إنشاء محركات أقراص قابلة للإقلاع من ملفات ISO القابلة للإقلاع (Windows و Linux وما إلى ذلك)","Създаване на стартиращи устройства от ISO образи (Windows, Linux и др.)","从可启动 ISO 文件 (Windows 和 Linux 等) 创建可启动驱动器","從可開機 ISO 檔案 (Windows 和 Linux 等) 建立可開機隨身碟","Stvaranje pogona za pokretanje iz ISO-ova za pokretanje (Windows, Linux itd.)","Vytváření bootovacích jednotek ze zaváděcích ISO (Windows, Linux atd.)","Lav opstartsbarer drev fra opstartsbarer ISOer (Window, Linux, osv.)","Opstartbare schijven aanmaken via opstartbare ISO's (Windows, Linux, enz)","Luo käynnistysasemia boottaavista ISO-kuvista (Windows, Linux jne.)","Créez des disques amorçables à partir d'images ISOs (Windows, Linux, etc.)","Erstellen bootfähiger Laufwerke aus bootfähigen ISOs (Windows, Linux, etc.)","Δημιουργήστε εκκινήσιμες μονάδες από ISO με δυνατότητα εκκίνησης (Windows, Linux, κ.λπ.)","יצירת כוננים הניתנים לאתחול מקובצי ISO הניתנים לאתחול (Windows‏, Linux וכו')","Bootolható meghajtók készítése bootolható ISO képfájlokból (Windows, Linux, stb.)","Buat perangkat yang dapat di boot dari ISO (Windows, Linux, dll.)","Crea unità avviabili da ISO avviabili (Windows, Linux, ecc.)","WindowsやLinuxなどのISOファイルから起動可能ドライブを作成します。","부팅 가능한 ISO (Windows, Linux 등)에서 부팅 가능한 드라이브 만들기","Izveido ielādes ierīces no ISO failiem (Windows, Linux, u.c.)","Sukurkite įkrovos diskus iš įkrovos ISO (Windows, Linux ir kt.)","Buat pemacu boleh boot daripada ISO boleh boot (Windows, Linux, dll.)","Lag oppstartabar enhet/disk fra ISOer (Windows, Linux, etc.)","ایجاد درایوهای قابل بوت از ISOهای قابل بوت (ویندوز، لینوکس و غیره)","Twórz dyski rozruchowe z obrazów ISO (Windows, Linux itp.)","Criar discos inicializáveis a partir de ISOs inicializáveis (Windows, Linux, etc.)","Criar unidades inicializáveis a partir de ISOs inicializáveis (Windows, Linux, etc.)","Crează drive-uri bootabile de la ISO-uri bootabile (Windows, Linux, etc.)","Создание загрузочных дисков из загрузочных образов ISO (Windows, Linux и т.д.)","Kreirajte disk jedinice za pokretanje sistema od ISO-a koji se mogu pokretanja sistema (Windows, Linux itd.)","Vytvorte bootovacie jednotky z ISO súborov (Windows, Linux atď.)","Ustvarite zagonske pogone iz zagonskih ISO-jev (Windows, Linux itd.)","Cree unidades de arranque desde ISO de arranque (Windows, Linux, etc.)","Skapa startbara enheter från startbara ISO-filer (Windows, Linux, etc.)","สร้าง USB ไดรฟ์บูตจากไฟล์ ISO ที่บูตได้ (เช่น ไฟล์ติดตั้ง Windows หรือ Linux เป็นต้น)","Önyüklenebilir ISO'lardan önyüklenebilir sürücüler oluşturun (Windows, Linux, vb.)","Створення завантажувальних пристроїв з завантажувальних образів (Windows, Linux, тощо)","Tạo ổ đĩa có thể khởi động từ các file ISO có thể khởi động (Windows, Linux, v.v...)" +"Feature4","703","Text","","Create bootable drives from bootable disk images, including compressed ones","إنشاء محركات أقراص قابلة للإقلاع من صور الأقراص القابلة للإقلاع ، بما في ذلك الأقراص المضغوطة","Създаване на стартиращи устройства от образи, включително компресирани такива","从可启动硬盘镜像 (包括压缩镜像) 创建可启动驱动器","從可開機硬碟映像檔 (包括壓縮映像檔) 建立可開機隨身碟","Stvaranje pogona za pokretanje iz slika diska za pokretanje, uključujući komprimirane","Vytváření zaváděcích jednotek ze zaváděcích obrazů disků, včetně komprimovaných","Lav opstartsbarer drev fra opstartsbarer disk billeder, inklusiv komprimerede billeder","Opstartbare schijven aanmaken van opstartbare schijf-images, inclusief gecomprimeerde images","Luo käynnistysasemia boottaavista levykuvista, pakatut kuvat mukaanlukien","Créez des disques amorçables à partir d'images disque, y compris à partir d'images compressées","Erstellen bootfähiger Laufwerke aus bootfähigen Festplatten-Images, einschließlich komprimierter Images","Δημιουργήστε μονάδες εκκίνησης από εικόνες δίσκου με δυνατότητα εκκίνησης, συμπεριλαμβανομένων συμπιεσμένων","יצירת כוננים הניתנים לאתחול מקובצי תמונת דיסק הניתנים לאתחול, כולל קבצים דחוסים","Bootolható meghajtók készítése bootolható lemez képfájlokból, beleértve a tömörítetteket is","Buat perangkat yang dapat di boot dari Disk Image, termasuk yang Disk Image yang terkompresi","Crea unità avviabili da immagini disco avviabili, incluse quelle compresse","圧縮済みのものを含むディスクイメージから起動可能ドライブを作成します。","압축된 이미지를 포함하여 부팅 가능한 디스크 이미지에서 부팅 가능한 드라이브 만들기","Izveido ielādes ierīces no ielādes disku virtuālajiem attēliem, tai skaitā arī no saspiestajiem","Sukurkite įkrovos diskus iš įkrovos disko vaizdų, įskaitant suspaustus","Buat pemacu boleh boot daripada imej cakera boleh boot, termasuk yang dimampatkan","Lag oppstartbare disker fra images, inkludert komprimerte sådan","درایوهای قابل بوت را از تصاویر دیسک قابل بوت، از جمله موارد فشرده، ایجاد کنید","Twórz dyski rozruchowe z obrazów dysków, włączając skompresowane","Criar discos inicializáveis a partir de imagens de disco inicializáveis, inclusive de imagens compactadas","Criar unidades inicializáveis a partir de imagens de disco inicializáveis, inclusive de imagens compactadas","Crează drive-uri de la imagini de disc bootabile, incluzând cele compresate","Создание загрузочных дисков из образов загрузочных дисков, в том числе сжатых","Kreiranje disk jedinica za pokretanje sistema sa slika diska koji se može pokretanja, uključujući kompresovane","Vytvorte bootovacie jednotky z diskových obrazov, vrátane tých komprimovaných","Ustvarite zagonske pogone iz slik diska, ki jih je mogoče zagnati, vključno s stisnjenimi","Cree unidades de arranque a partir de imágenes de disco de arranque, incluidas las comprimidas","Skapa startbara enheter från startbara diskavbildningar, inklusive komprimerade","สร้างไดรฟ์บูตจากดิสก์อิมเมจที่บูตได้ (รองรับไฟล์ดิสก์อิมเมจที่ถูกบีบอัด)","Sıkıştırılmış olanlar da dahil olmak üzere önyüklenebilir disk yansısından önyüklenebilir sürücüler oluşturun","Створення завантажувальних пристроїв з завантажувальних образів, у тому числі стиснутих","Tạo ổ đĩa có thể khởi động từ các tệp đĩa có thể khởi động, bao gồm cả tệp nén" +"Feature5","704","Text","","Create BIOS or UEFI bootable drives, including UEFI bootable NTFS","إنشاء BIOS أو محركات أقراص UEFI قابلة للإقلاع ، بما في ذلك UEFI NTFS القابل للتشغيل","Създаване на BIOS или UEFI стартиращи устройства, включително UEFI стартиращ NTFS","创建 BIOS 或 UEFI 可启动驱动器，包括 UEFI 可启动的 NTFS 驱动器","建立 BIOS 或 UEFI 可開機隨身碟，包括 UEFI 下的可開機 NTFS 隨身碟","Stvaranje BIOS ili UEFI pogona za pokretanje, uključujući UEFI NTFS za pokretanje","Vytvořte zaváděcí jednotky BIOS nebo UEFI, včetně zaváděcích souborů NTFS UEFI","Lav BIOS eller UEFI opstartsbarer drev, inklusiv UEFI opstartsbarer NTFS","BIOS of UEFI opstartbare schijven aanmaken, inclusief UEFI opstartbare NTFS","Luo BIOS- tai UEFI-boottaavia asemia, mukaanlukien UEFI-boottaavat NTFS-asemat","Créez des disques amorçables BIOS ou UEFI, y compris des disques UEFI amorçables utilisant NTFS","Erstellen von BIOS- oder UEFI-bootfähigen Laufwerken, einschließlich UEFI-bootfähigem NTFS","Δημιουργήστε μονάδες δίσκου με δυνατότητα εκκίνησης BIOS ή UEFI, συμπεριλαμβανομένων NTFS με δυνατότητα εκκίνησης UEFI","יצירת כוננים הניתנים לאתחול ממחשבים התומכים ב־BIOS או UEFI, לרבות כוננים הניתנים לאתחול מ־UEFI, המשתמשים במערכת הקבצים NTFS","BIOS-ból vagy UEFI-ből bootolható meghajtók készítése, beleértve az UEFI-ből bootolható NTFS meghajtókat is","Buat perangkat BIOS atau UEFI yang dapat di boot, termasuk perangkat NTFS yang dapat di boot oleh UEFI","Crea unità avviabili BIOS o UEFI, incluso NTFS avviabile UEFI","UEFI:NTFSを含むBIOS及びUEFIで起動可能なドライブを作成します。","UEFI 부팅 가능한 NTFS를 포함하여 BIOS 또는 UEFI 부팅 가능 드라이브 만들기","Izveido BIOS vai UEFI ielādes ierīces, ieskaitot UEFI ielādi no NTFS","Sukurkite BIOS arba UEFI įkrovos diskus, įskaitant UEFI įkrovos NTFS","Buat pemacu boleh boot BIOS atau UEFI, termasuk NTFS boleh boot UEFI","Lag BIOS eller UEFI oppstartbare disker, inkludert UEFI oppstartbar NTFS","درایوهای قابل بوت بایوس یا UEFI از جمله NTFS قابل بوت UEFI ایجاد کنید","Stwórz dysk rozruchowy BIOS lub UEFI, włączając bootowalny dysk UEFI NTFS","Criar discos inicializáveis ​​BIOS ou UEFI, inclusive discos UEFI inicializáveis ​​usando NTFS","Criar discos inicializáveis ​​BIOS ou UEFI, inclusive discos UEFI inicializáveis ​​usando NTFS","Crează discuri bootabile BIOS sau UEFI, incluzând NTFS-uri bootabile de UEFI","Создание загрузочных дисков BIOS или UEFI, включая загрузочный UEFI NTFS","Kreiranje BIOS ili UEFI disk jedinica za pokretanje sistema, uključujući NTFS sa UEFI pokretanjem sistema","Vytvorte bootovacie jednotky systému BIOS alebo UEFI vrátane UEFI bootovateľnej jednotky NTFS","Ustvarite zagonske pogone BIOS ali UEFI, vključno z UEFI bootable NTFS","Cree unidades de arranque BIOS o UEFI, incluido NTFS de arranque UEFI","Skapa BIOS- eller UEFI-startbara enheter, inklusive UEFI-startbar NTFS","สร้างไดรฟ์บูตสำหรับระบบ BIOS หรือ UEFI และ UEFI bootable NTFS","UEFI önyüklenebilir NTFS dahil BIOS ya da UEFI önyüklenebilir sürücüler oluşturun","Створення завантажувальних пристроїв BIOS чи UEFI, включаючи завантажувальний UEFI NTFS","Tạo ổ đĩa có thể khởi động với hệ thống BIOS hoặc UEFI, bao gồm cả NTFS có thể khởi động với UEFI" "Feature6","705","Text","","Create 'Windows To Go' drives","إنشاء محركات أقراص ""Windows To Go\","Създаване на 'Windows To Go' устройства","创建 'Windows To Go' 驱动器","建立 'Windows To Go' 隨身碟","Stvaranje pogona ""Windows To Go\","Vytvoření jednotek ""Windows To Go","Lav 'Windows To Go' drev","'Windows To Go'-schijven aanmaken","Luo 'Windows To Go' -asemia","Créez des disques 'Windows To Go'","Erstellen von ""Windows To Go""-Laufwerken","Δημιουργήστε μονάδες δίσκου ""Windows To Go\","יצירת כונני Windows To Go","'Windows To Go' meghajtók készítése","Buat perangkat Windows To Go","Crea unità 'Windows To Go'","Windows To Goドライブを作成します。","'Windows To Go' 드라이브 만들기","Izveido 'Windows To Go' ierīces","Sukurkite ""Windows To Go"" diskus","Buat pemacu 'Windows To Go'","Lag 'Windows To Go' disker","درایوهای ""Windows To Go"" را ایجاد کنید","Stwórz dysk 'Windows To Go'","Criar discos 'Windows To Go'","Criar discos 'Windows To Go'","Crează discuri 'Windows To Go'","Создание дисков Windows To Go","Kreiranje disk jedinica ""Windows to Go\","Vytvorte jednotky „Windows To Go\","Ustvarjanje pogonov »Windows To Go'","Cree unidades 'Windows To Go'","Skapa ""Windows To Go""-enheter","สร้างไดรฟ์ของ 'Windows To Go'","'Windows To Go' sürücüleri oluşturun","Створення пристроїв 'Windows To Go'","Tạo ổ đĩa 'Windows To Go'" "Feature7","706","Text","","Create Windows 11 installation drives for PCs that don't have TPM or Secure Boot","قم بإنشاء محركات تثبيت ويندوز 11 لأجهزة الكمبيوتر التي لا تحتوي على TPM أو الإقلاع الآمن (Secure Boot)","Създаване на Windows 11 инсталационно устройство за компютри, които нямат TPM или Secure Boot","为没有 TPM 或安全启动功能的电脑创建 Windows 11 安装驱动器","為沒有 TPM 或安全開機功能的電腦建立 Windows 11 安裝隨身碟","Stvaranje instalacijskih pogona sustava Windows 11 za PC-jeve koji nemaju TPM ili Sigurno pokretanje","Vytvoření instalačních jednotek systému Windows 11 pro počítače bez čipu TPM nebo Secure Boot","Lav Windows 11 installations drev for PCer der ikker har TPM eller Secure Boot","Windows 11 installatieschijven aanmaken voor pc's die geen TPM of Secure Boot hebben","Luo Windows 11 -asennusasemia tietokoneille, jotka eivät tue TPM- tai Secure Boot -ominaisuuksia","Créez des disques d'installation Windows 11 pour des PCs qui ne disposent pas de TPM ou Secure Boot","Erstellen von Windows 11-Installationslaufwerken für PCs ohne TPM oder Secure Boot","Δημιουργήστε μονάδες εγκατάστασης των Windows 11 για υπολογιστές που δεν διαθέτουν TPM ή Ασφαλή Εκκίνηση","יצירת כונני התקנה של Windows 11 עבור מחשבים שאין להם TPM או Secure Boot","Windows 11 telepítési meghajtók készítése olyan PC-k számára, amelyek nem rendelkeznek TPM vagy Secure Boot funkciókkal","Buat perangkat pemasang Windows 11 untuk PC yang tidak mempunyai TPM atau Secure Boot","Crea unità di installazione di Windows 11 per PC che non dispongono di TPM o avvio protetto","TPM及びセキュアブート非対応のPC向けのWindows 11インストールドライブを作成します","TPM 또는 보안 부팅이 없는 PC용 Windows 11 설치 드라이브 만들기","Izveido Windows 11 instalācijas ierīces datoriem, kam nav TPM vai Secure Boot","Sukurkite Windows 11 diegimo diskus kompiuteriams, kuriuose nėra TPM arba saugaus įkrovimo","Buat pemacu pemasangan Windows 11 untuk PC yang tidak mempunyai TPM atau But Selamat","Lag oppstartsmedia for Windows 11 som ikke krever TPM eller Secure Boot","درایوهای نصب ویندوز 11 را برای رایانه هایی که TPM یا Secure Boot ندارند ایجاد کنید","Twórz dyski instalacyjne systemu Windows 11 dla komputerów, które nie posiadają modułu TPM ani Secure Boot","Criar discos de instalação do Windows 11 para PCs que não possuem TPM ou Secure Boot","Criar discos de instalação do Windows 11 para PCs que não possuem TPM ou Arranque Seguro","Crează drive de instalare Windows 11 pe computere care nu au TPM sau Bootare Securizată","Создание установочных дисков Windows 11 для компьютеров без TPM или безопасной загрузки","Kreiranje windows 11 instalacionih disk jedinica za računare koji nemaju TPM ili bezbedno pokretanje sistema","Vytvorte inštalačné jednotky Windows 11 pre počítače, ktoré nemajú modul TPM, ani Secure Boot","Ustvarjanje namestitvenih pogonov za Windows 11 za računalnike, ki nimate TPM ali Varnega zagona","Cree unidades de instalación de Windows 11 para PC que no tienen TPM o Arranque seguro","Skapa installationsenheter till Windows 11 för datorer som inte har TPM eller säker start","สร้างไดรฟ์ติดตั้ง Windows 11 สำหรับคอมพิวเตอร์ที่ไม่มี TPM หรือ Secure Boot","TPM ya da Güvenli Önyüklemeye sahip olmayan bilgisayarlar için Windows 11 kurulum sürücüleri oluşturun","Створення інсталяційних дисків Windows 11 для ПК, які не мають TPM чи Secure Boot","Tạo ổ đĩa cài đặt Windows 11 cho các máy tính không có TPM hoặc Secure Boot" -"Feature8","707","Text","","Create persistent Linux partitions","إنشاء Persistent Linux partitions","Създаване на устойчиви Linux дялове","创建持久 Linux 分区","建立持續性 Linux 磁區","Stvaranje trajnih Linux particija","Vytvoření trvalých oddílů systému Linux","Lav vedvarende Linux adskillelser","Persistent Linux partities aanmaken","Luo pysyviä Linux-osioita","Créez des partitions persistentes pour Linux","Persistente Linux-Partitionen erstellen","Δημιουργήστε μόνιμα διαμερίσματα Linux","יצירת מחיצות Linux קבועות","Tartós Linux partíciók készítése","Buat partisi Linux yang tetap/persistent","Crea partizioni persistenti Linux","記録用Linuxパーティションを作成します。","영구 리눅스 파티션 만들기","Izveido pastāvīgas Linux partīcijas","Sukurkite nuolatinius Linux skaidinius","Buat partition Linux berterusan","Lag persistente Linux partisjoner","ایجاد پارتیشن های لینوکس دائمی","Stwórz particje persistent Linuxa","Criar partições persistentes para Linux","Crie partições persistentes para Linux","Crează partiție de Linux persistentă","Создание постоянных разделов Linux","Kreiranje upornih Linux particija","Vytvorte trvalé oblasti systému Linux","Ustvarjanje trajnih Linux particij","Crear particiones persistentes de Linux","Skapa beständiga Linux-partitioner","สร้าง Linux partition แบบ persistent","Kalıcı Linux bölümleri oluşturun","Створення розділу збереження Linux","Tạo phân vùng Linux liên tục" -"Feature9","708","Text","","Create VHD/DD images of the selected drive","إنشاء صور VHD / DD لمحرك الأقراص المحدد","Създаване на VHD/DD образи на избраното устройство","为选择的驱动器创建 VHD/DD 镜像","為選中的磁碟機建立 VHD/DD 映像檔","Stvaranje VHD/DD slika odabranog pogona","Vytvoření obrazů VHD/DD z vybrané jednotky","Lav VHD/DD billeder af det valgte drev","VHD/DD-images van de geselecteerde schijf aanmaken","Luo VHD/DD-kuvia valitusta asemasta","Créez des images VHD/DD du périphérique sélectionné","VHD/DD-Images des ausgewählten Laufwerks erstellen","Δημιουργήστε εικόνες VHD/DD της επιλεγμένης μονάδας δίσκου","יצירת קובצי תמונה מסוג VHD/DD של הכונן שנבחר","VHD/DD képfájl készítése a kiválasztott meghajtóról","Buat image VHD/DD dari penyimpanan yang dipilih","Crea immagini VHD/DD dell'unità selezionata","選択されたドライブのVHD/DDイメージを作成します。","선택한 드라이브의 VHD/DD 이미지 만들기","Izveido izvēlētā diska VHD/DD virtuālos attēlus","Sukurkite pasirinkto disko VHD / DD vaizdus","Buat imej VHD/DD bagi pemacu yang dipilih","Lag VHD/DD speilinger av valgt disk","تصاویر VHD/DD از درایو انتخاب شده ایجاد کنید","Stwórz obraz VHD/DD z wybranego dysku","Criar imagens VHD/DD do dispositivo selecionado","Criar imagens VHD/DD do dispositivo selecionado","Crează imagini VHD/DD de pe drive-ul selectat","Создание образов VHD/DD выбранного диска","Kreiranje VHD/DD slika izabrane disk jedinice","Vytvorte obrazy VHD/DD z vybratej jednotky","Ustvarjanje VHD/DD slik izbranega pogona","Cree imágenes VHD/DD de la unidad seleccionada","Skapa VHD/DD-avbilder av den valda enheten","สร้างอิมเมจไฟล์แบบ VHD/DD จากไดรฟ์ที่เลือก","Seçilen sürücünün VHD/DD yansılarını oluşturun","Створення образів VHD/DD з вибраних дисків","Tạo tệp VHD/DD từ ổ đĩa đã chọn" -"Feature10","709","Text","","Compute MD5, SHA-1, SHA-256 and SHA-512 checksums of the selected image","حساب المجاميع الإختبارية MD5 و SHA-1 و SHA-256 و SHA-512 للصورة المحددة","Изчисляване на MD5, SHA-1, SHA-256 и SHA-512 чексуми на избраният образ","计算被选择镜像的 MD5、SHA-1、SHA-256 和 SHA-512 校验码","計算被選中映像的 MD5、SHA-1、SHA-256 和 SHA-512 檢查碼","Izračunajte kontrolne zbrojeve odabrane slike MD5, SHA-1, SHA-256 i SHA-512","Výpočet kontrolních součtů MD5, SHA-1, SHA-256 a SHA-512 vybraného obrazu","Beregn MD5, SHA-1, SHA-256 og SHA-512 checksums af det valgte billede","MD5, SHA-1, SHA-256 en SHA-512 controlesommen berekenen van de geselecteerde image","Laske MD5, SHA-1, SHA-256 ja SHA-512 tarkistussummia valitusta levykuvasta","Calculez les sommes de contrôle MD5, SHA-1, SHA-256 et SHA-512 de l'image sélectionnée","Berechnung von MD5-, SHA-1-, SHA-256- und SHA-512-Prüfsummen für das ausgewählte Bild","Υπολογίστε τα αθροίσματα ελέγχου MD5, SHA-1, SHA-256 και SHA-512 της επιλεγμένης εικόνας","חישוב סיכומי ביקורת מסוג MD5,‏ SHA-1,‏ SHA-256 ו־SHA-512 של קובץ התמונה שנבחרה","A kiválasztott képfájl MD5, SHA-1, SHA-256 és SHA-512 ellenőrző összegének kiszámítása","Hitung checksum MD5, SHA-1, SHA-256 dan SHA-512 dari image yang dipilih","Calcola i checksum MD5, SHA-1, SHA-256 e SHA-512 dell'immagine selezionata","選択されたイメージのMD5、SHA-1、SHA-256及びSHA-512チェックサムを計算します。","선택한 이미지의 MD5, SHA-1, SHA-256 및 SHA-512 체크섬 계산","Izskaitļo izvēlētā virtuālā attēla MD5, SHA-1, SHA-256 un SHA-512 kontrolsummas","Apskaičiuokite pasirinkto vaizdo MD5, SHA-1, SHA-256 ir SHA-512 kontrolines sumas","Kira MD5, SHA-1, SHA-256 dan SHA-512 checksum imej yang dipilih","Kalkuler MD5, SHA-1, SHA-256 og SHA-512 sjekksummer fra valgt speiling","MD5، SHA-1، SHA-256 و SHA-512 را برای تصویر انتخابی محاسبه کنید","Oblicz sumy kontrolne MD5, SHA-1, SHA-256 i SHA-512 dla wybranego obrazu","Calcular somas de verificação MD5, SHA-1, SHA-256 e SHA-512 da imagem selecionada","Calcular checksums MD5, SHA-1, SHA-256 e SHA-512 da imagem selecionada","Compută MD5, SHA-1, SHA-256 și SHA-512 suma de control ale imaginilor selectate","Вычисление контрольных сумм MD5, SHA-1, SHA-256 и SHA-512 выбранного образа","Izračunajte MD5, SHA-1, SHA-256 i SHA-512 kontrolne preglede izabrane slike","Vypočítajte kontrolné súčty vybratého obrazu (MD5, SHA-1, SHA-256 a SHA-512)","Račun MD5, SHA-1, SHA-256 in SHA-512 kontrolni vsoti izbrane slike","Calcule las sumas de comprobación MD5, SHA-1, SHA-256 y SHA-512 de la imagen seleccionada","Beräkna kontrollsummor MD5, SHA-1, SHA-256 och SHA-512 för den valda avbilden","คำนวณรหัส MD5, SHA-1, SHA-256, SHA-512 ของไฟล์อิมเมจที่เลือก","Seçilen yansının MD5, SHA-1, SHA-256 ve SHA-512 sağlama toplamlarını hesaplayın","Обчислення контрольних сум MD5, SHA-1, SHA-256 та SHA-512 для вибраних образів","Tính tổng kiểm MD5, SHA-1, SHA-256 và SHA-512 của tệp đã chọn" -"Feature11","710","Text","","Perform bad blocks checks, including detection of ""fake"" flash drives","إجراء فحوصات كتل تالفة ، بما في ذلك الكشف عن محركات أقراص فلاش ""زائفة\","Проверяване за лоши блокове, включително засичане на ""фалшиви"" устройства","执行坏块检查，包括对”假“U盘的检测","執行壞軌檢查，包括對”假“USB 快閃磁碟機的偵測","Izvršite provjere loših blokova, uključujući otkrivanje ""lažnih"" flash pogona","Provést kontrolu vadných bloků, včetně detekce ""falešných"" bloků. flash disky","Udøv dårlige blokke tjeks, inklusiv opdagelse af ""falske"" flashdrev","Controles uitvoeren op slechte blokken, inclusief detectie van ""valse"" flashdrives","Suorita viallisten lohkojen tarkistuksia, sisältäen ""valheellisten"" muistitikkujen tunnistamisen","Executez un test de mauvais secteurs avec detection des ""fake drives\","Durchführung von Prüfungen auf fehlerhafte Blöcke, einschließlich der Erkennung von ""gefälschten"" Flash-Laufwerken","Εκτελέστε ελέγχους εσφαλμένων block, συμπεριλαμβανομένου του εντοπισμού ""ψευδών"" μονάδων flash","ביצוע בדיקות אחר בלוקים (אזורים) פגומים, כולל זיהוי של כונני הבזק ""מזוייפים\","Hibás blokkok ellenőrzése, beleértve a ""hamis"" flash meghajtók detektálását","Lakukan cek Blok yang buruk, termasuk deteksi USB Flash Disk ""PALSU\","Esegui controlli dei blocchi danneggiati, incluso il rilevamento di unità flash ""false\","不良ブロックチェック及び容量詐欺ドライブの検知を行います。","""위조"" 플래시 드라이브 감지를 포함하여 불량 블록 검사 수행","Izpilda bojāto bloku pārbaudi ieskaitot ""falsificēto"" nesēju noteikšanu","Atlikite blogų blokų patikrinimus, įskaitant ""netikrų"" flash diskų aptikimą","Melakukan pemeriksaan blok buruk, termasuk pengesanan pemacu kilat ""palsu\","Utfør sjekk for dårlige sektorer, inkludert sjekk for forfalskede flash disker","بررسی بلوک های بد، از جمله تشخیص درایوهای فلش ""جعلی"" را انجام دهید","Sprawdź dysk pod względem spójności danych lub wykryj ""nieorginalny"" pendrive","Executar verificações de blocos defeituosos, incluindo detecção de unidades flash ""falsificadas\","Executar verificações de blocos inválidos, incluindo a deteção de unidades flash ""falsas\","Verifică blocuri rele, incluzând detectarea de drive-uri flash ""false\","Проверка на плохие блоки, обнаружение ""поддельных"" флешек","Izvršite provere loših blokova, uključujući otkrivanje ""lažnih"" fleš diskova","Vykonajte kontroly zlých blokov vrátane detekcie „falošných"" prenosných diskov","Izvajanje preverjanj slabih blokov, vključno z odkrivanjem »lažnih« bliskavic","Realice comprobaciones de bloques defectuosos, incluida la detección de unidades flash ""falsas\","Utför kontroll av trasiga block, inklusive upptäckt av ""falska"" USB-minnen","ตรวจสอบจุดบกพร่อง รวมไปถึงการทดสอบแฟลชไดรฟ์ว่าเป็น ""ของปลอม"" หรือไม่","""Sahte"" flash sürücülerin tespiti de dahil olmak üzere hatalı blok kontrolleri gerçekleştirin","Перевірка дисків (включаючи фальшиві диски)","Thực hiện kiểm tra điểm lỗi, bao gồm phát hiện ổ đĩa ""giả\" +"Feature8","707","Text","","Create persistent Linux partitions","إنشاء Persistent Linux partitions","Създаване на устойчиви Linux дялове","创建持久 Linux 分区","建立持續性 Linux 磁區","Stvaranje trajnih Linux particija","Vytvoření trvalých oddílů systému Linux","Lav vedvarende Linux adskillelser","Persistent Linux partities aanmaken","Luo pysyviä Linux-osioita","Créez des partitions persistentes pour Linux","Persistente Linux-Partitionen erstellen","Δημιουργήστε μόνιμα διαμερίσματα Linux","יצירת מחיצות Linux קבועות","Tartós Linux partíciók készítése","Buat partisi Linux yang tetap/persistent","Crea partizioni persistenti Linux","記録用Linuxパーティションを作成します。","영구 리눅스 파티션 만들기","Izveido pastāvīgas Linux partīcijas","Sukurkite nuolatinius Linux skaidinius","Buat partition Linux berterusan","Lag persistente Linux partisjoner","ایجاد پارتیشن های لینوکس دائمی","Utwórz trwałe partycje Linux","Criar partições persistentes para Linux","Crie partições persistentes para Linux","Crează partiție de Linux persistentă","Создание постоянных разделов Linux","Kreiranje upornih Linux particija","Vytvorte trvalé oblasti systému Linux","Ustvarjanje trajnih Linux particij","Crear particiones persistentes de Linux","Skapa beständiga Linux-partitioner","สร้าง Linux partition แบบ persistent","Kalıcı Linux bölümleri oluşturun","Створення розділу збереження Linux","Tạo phân vùng Linux liên tục" +"Feature9","708","Text","","Create VHD/DD images of the selected drive","إنشاء صور VHD / DD لمحرك الأقراص المحدد","Създаване на VHD/DD образи на избраното устройство","为选择的驱动器创建 VHD/DD 镜像","為選取的磁碟機建立 VHD/DD 映像檔","Stvaranje VHD/DD slika odabranog pogona","Vytvoření obrazů VHD/DD z vybrané jednotky","Lav VHD/DD billeder af det valgte drev","VHD/DD-images van de geselecteerde schijf aanmaken","Luo VHD/DD-kuvia valitusta asemasta","Créez des images VHD/DD du périphérique sélectionné","VHD/DD-Images des ausgewählten Laufwerks erstellen","Δημιουργήστε εικόνες VHD/DD της επιλεγμένης μονάδας δίσκου","יצירת קובצי תמונה מסוג VHD/DD של הכונן שנבחר","VHD/DD képfájl készítése a kiválasztott meghajtóról","Buat image VHD/DD dari penyimpanan yang dipilih","Crea immagini VHD/DD dell'unità selezionata","選択されたドライブのVHD/DDイメージを作成します。","선택한 드라이브의 VHD/DD 이미지 만들기","Izveido izvēlētā diska VHD/DD virtuālos attēlus","Sukurkite pasirinkto disko VHD / DD vaizdus","Buat imej VHD/DD bagi pemacu yang dipilih","Lag VHD/DD speilinger av valgt disk","تصاویر VHD/DD از درایو انتخاب شده ایجاد کنید","Stwórz obraz VHD/DD z wybranego dysku","Criar imagens VHD/DD do dispositivo selecionado","Criar imagens VHD/DD do dispositivo selecionado","Crează imagini VHD/DD de pe drive-ul selectat","Создание образов VHD/DD выбранного диска","Kreiranje VHD/DD slika izabrane disk jedinice","Vytvorte obrazy VHD/DD z vybratej jednotky","Ustvarjanje VHD/DD slik izbranega pogona","Cree imágenes VHD/DD de la unidad seleccionada","Skapa VHD/DD-avbilder av den valda enheten","สร้างอิมเมจไฟล์แบบ VHD/DD จากไดรฟ์ที่เลือก","Seçilen sürücünün VHD/DD yansılarını oluşturun","Створення образів VHD/DD з вибраних дисків","Tạo tệp VHD/DD từ ổ đĩa đã chọn" +"Feature10","709","Text","","Compute MD5, SHA-1, SHA-256 and SHA-512 checksums of the selected image","حساب المجاميع الإختبارية MD5 و SHA-1 و SHA-256 و SHA-512 للصورة المحددة","Изчисляване на MD5, SHA-1, SHA-256 и SHA-512 чексуми на избраният образ","计算被选择镜像的 MD5、SHA-1、SHA-256 和 SHA-512 校验码","計算被選取映像的 MD5、SHA-1、SHA-256 和 SHA-512 檢查碼","Izračunajte kontrolne zbrojeve odabrane slike MD5, SHA-1, SHA-256 i SHA-512","Výpočet kontrolních součtů MD5, SHA-1, SHA-256 a SHA-512 vybraného obrazu","Beregn MD5, SHA-1, SHA-256 og SHA-512 checksums af det valgte billede","MD5, SHA-1, SHA-256 en SHA-512 controlesommen berekenen van de geselecteerde image","Laske MD5, SHA-1, SHA-256 ja SHA-512 tarkistussummia valitusta levykuvasta","Calculez les sommes de contrôle MD5, SHA-1, SHA-256 et SHA-512 de l'image sélectionnée","Berechnung von MD5-, SHA-1-, SHA-256- und SHA-512-Prüfsummen für das ausgewählte Bild","Υπολογίστε τα αθροίσματα ελέγχου MD5, SHA-1, SHA-256 και SHA-512 της επιλεγμένης εικόνας","חישוב סיכומי ביקורת מסוג MD5,‏ SHA-1,‏ SHA-256 ו־SHA-512 של קובץ התמונה שנבחרה","A kiválasztott képfájl MD5, SHA-1, SHA-256 és SHA-512 ellenőrző összegének kiszámítása","Hitung checksum MD5, SHA-1, SHA-256 dan SHA-512 dari image yang dipilih","Calcola i checksum MD5, SHA-1, SHA-256 e SHA-512 dell'immagine selezionata","選択されたイメージのMD5、SHA-1、SHA-256及びSHA-512チェックサムを計算します。","선택한 이미지의 MD5, SHA-1, SHA-256 및 SHA-512 체크섬 계산","Izskaitļo izvēlētā virtuālā attēla MD5, SHA-1, SHA-256 un SHA-512 kontrolsummas","Apskaičiuokite pasirinkto vaizdo MD5, SHA-1, SHA-256 ir SHA-512 kontrolines sumas","Kira MD5, SHA-1, SHA-256 dan SHA-512 checksum imej yang dipilih","Kalkuler MD5, SHA-1, SHA-256 og SHA-512 sjekksummer fra valgt speiling","MD5، SHA-1، SHA-256 و SHA-512 را برای تصویر انتخابی محاسبه کنید","Oblicz sumy kontrolne MD5, SHA-1, SHA-256 i SHA-512 dla wybranego obrazu","Calcular somas de verificação MD5, SHA-1, SHA-256 e SHA-512 da imagem selecionada","Calcular checksums MD5, SHA-1, SHA-256 e SHA-512 da imagem selecionada","Compută MD5, SHA-1, SHA-256 și SHA-512 suma de control ale imaginilor selectate","Вычисление контрольных сумм MD5, SHA-1, SHA-256 и SHA-512 выбранного образа","Izračunajte MD5, SHA-1, SHA-256 i SHA-512 kontrolne preglede izabrane slike","Vypočítajte kontrolné súčty vybratého obrazu (MD5, SHA-1, SHA-256 a SHA-512)","Račun MD5, SHA-1, SHA-256 in SHA-512 kontrolni vsoti izbrane slike","Calcule las sumas de comprobación MD5, SHA-1, SHA-256 y SHA-512 de la imagen seleccionada","Beräkna kontrollsummor MD5, SHA-1, SHA-256 och SHA-512 för den valda avbilden","คำนวณรหัส MD5, SHA-1, SHA-256, SHA-512 ของไฟล์อิมเมจที่เลือก","Seçilen yansının MD5, SHA-1, SHA-256 ve SHA-512 sağlama toplamlarını hesaplayın","Обчислення контрольних сум MD5, SHA-1, SHA-256 та SHA-512 для вибраних образів","Tính tổng kiểm MD5, SHA-1, SHA-256 và SHA-512 của tệp đã chọn" +"Feature11","710","Text","","Perform bad blocks checks, including detection of ""fake"" flash drives","إجراء فحوصات كتل تالفة ، بما في ذلك الكشف عن محركات أقراص فلاش ""زائفة\","Проверяване за лоши блокове, включително засичане на ""фалшиви"" устройства","执行坏块检查，包括对”假“U盘的检测","執行壞軌檢查，包括對”假“USB 快閃磁碟 +的偵測","Izvršite provjere loših blokova, uključujući otkrivanje ""lažnih"" flash pogona","Provést kontrolu vadných bloků, včetně detekce ""falešných"" bloků. flash disky","Udøv dårlige blokke tjeks, inklusiv opdagelse af ""falske"" flashdrev","Controles uitvoeren op slechte blokken, inclusief detectie van ""valse"" flashdrives","Suorita viallisten lohkojen tarkistuksia, sisältäen ""valheellisten"" muistitikkujen tunnistamisen","Executez un test de mauvais secteurs avec detection des ""fake drives\","Durchführung von Prüfungen auf fehlerhafte Blöcke, einschließlich der Erkennung von ""gefälschten"" Flash-Laufwerken","Εκτελέστε ελέγχους εσφαλμένων block, συμπεριλαμβανομένου του εντοπισμού ""ψευδών"" μονάδων flash","ביצוע בדיקות אחר בלוקים (אזורים) פגומים, כולל זיהוי של כונני הבזק ""מזוייפים\","Hibás blokkok ellenőrzése, beleértve a ""hamis"" flash meghajtók detektálását","Lakukan cek Blok yang buruk, termasuk deteksi USB Flash Disk ""PALSU\","Esegui controlli dei blocchi danneggiati, incluso il rilevamento di unità flash ""false\","不良ブロックチェック及び容量詐欺ドライブの検知を行います。","""위조"" 플래시 드라이브 감지를 포함하여 불량 블록 검사 수행","Izpilda bojāto bloku pārbaudi ieskaitot ""falsificēto"" nesēju noteikšanu","Atlikite blogų blokų patikrinimus, įskaitant ""netikrų"" flash diskų aptikimą","Melakukan pemeriksaan blok buruk, termasuk pengesanan pemacu kilat ""palsu\","Utfør sjekk for dårlige sektorer, inkludert sjekk for forfalskede flash disker","بررسی بلوک های بد، از جمله تشخیص درایوهای فلش ""جعلی"" را انجام دهید","Sprawdź dysk pod względem spójności danych lub wykryj ""nieoryginalny"" pendrive","Executar verificações de blocos defeituosos, incluindo detecção de unidades flash ""falsificadas\","Executar verificações de blocos inválidos, incluindo a deteção de unidades flash ""falsas\","Verifică blocuri rele, incluzând detectarea de drive-uri flash ""false\","Проверка на плохие блоки, обнаружение ""поддельных"" флешек","Izvršite provere loših blokova, uključujući otkrivanje ""lažnih"" fleš diskova","Vykonajte kontroly zlých blokov vrátane detekcie „falošných"" prenosných diskov","Izvajanje preverjanj slabih blokov, vključno z odkrivanjem »lažnih« bliskavic","Realice comprobaciones de bloques defectuosos, incluida la detección de unidades flash ""falsas\","Utför kontroll av trasiga block, inklusive upptäckt av ""falska"" USB-minnen","ตรวจสอบจุดบกพร่อง รวมไปถึงการทดสอบแฟลชไดรฟ์ว่าเป็น ""ของปลอม"" หรือไม่","""Sahte"" flash sürücülerin tespiti de dahil olmak üzere hatalı blok kontrolleri gerçekleştirin","Перевірка дисків (включаючи фальшиві диски)","Thực hiện kiểm tra điểm lỗi, bao gồm phát hiện ổ đĩa ""giả\" "Feature12","711","Text","","Download official Microsoft Windows retail ISOs","قم بتنزيل ملفات ISO الرسمية الخاصة بـ Microsoft Windows","Изтегляне на официални Microsoft Windows ISO образи","下载微软官方 Windows 镜像","下載微軟官方 Windows 映像檔","Preuzimanje službenih ISO-ova za maloprodaju sustava Microsoft Windows","Stažení oficiálních souborů ISO systému Microsoft Windows","Hent officielle Microsoft Windows detail ISOer","Officiële Microsoft Windows retail ISO's downloaden","Lataa virallisia Microsoft Windowsin jälleenmyyntiversion ISO-levykuvia","Téléchargez des images ISOs commerciales officielles de Microsoft Windows","Offizielle Microsoft Windows-ISOs herunterladen","Κατεβάστε τα επίσημα retail ISO των Microsoft Windows","הורדת קובצי ה־ISO הקמעונאיים הרשמיים של Microsoft Windows","Hivatalos Microsoft Windows kiskereskedelmi ISO képfájlok letöltése","Unduh ISO Microsoft Windows resmi","Scarica le ISO ufficiali di Microsoft Windows","マイクロソフト公式のWindows ISOをダウンロードします。","공식 Microsoft Windows 리테일 ISO 다운로드","Lejupielādē oficiālos Microsoft ISO failus","Atsisiųskite oficialius Microsoft Windows mažmeninės prekybos ISO","Muat turun rasmi ISO runcit Microsoft Windows","Last ned offisielle Windows ISOer","ISO های رسمی ماکروسافت را دریافت کنید","Pobierz oficjalny obraz ISO systemu Microsoft Windows","Baixar ISOs oficiais do Microsoft Windows","Transferir ISOs oficiais do Microsoft Windows Retail","Descarcă un Microsoft Windows ISO oficial de vânzare","Загрузка официальных ISO-образов Windows","Preuzmite zvanične Microsoft Windows maloprodajne ISO-ove","Stiahnite si oficiálne ISO pre Microsoft Windows","Prenos uradnih Microsoft Windows maloprodaja ISOs","Descargue los ISO oficiales de Microsoft Windows","Ladda ner officiella Microsoft Windows ISO-filer","ดาวน์โหลดไฟล์ ISO ของ Microsoft Windows จากเว็บไซต์ทางการของ Microsoft","Resmi Microsoft Windows Retail ISO'larını indirin","Завантаження офіційних образів Microsoft Windows","Tải xuống các tệp Microsoft Windows ISO bán lẻ chính thức" "Feature13","712","Text","","Download UEFI Shell ISOs","قم بتنزيل UEFI Shell ISOs","Изтегляне на UEFI Shell образи","下载 UEFI Shell 镜像","下載 UEFI Shell 映像檔","Preuzmite ISO-ove UEFI ljuske","Stažení souborů UEFI Shell ISO","Hent UEFI Shell ISOer","UEFI Shell ISO's downloaden","Lataa UEFI Shell ISO-levykuvia","Téléchargez des images ISOs du Shell UEFI","UEFI-Shell-ISOs herunterladen","Κατεβάστε τα ISO Shell UEFI","הורדת קובצי ISO של מעטפת UEFI","UEFI Shell ISO képfájlok letöltése","Unduh Shell ISO UEFI","Scarica le ISO della shell UEFI","UEFIシェルのISOをダウンロードします。","UEFI Shell ISO 다운로드","Lejupielādē UEFI OS ISO failus","Atsisiųskite UEFI Shell ISO","Muat turun ISO Shell UEFI","Last ned UEFI kommandolinje ISOer","ISO های پوسته UEFI را دانلود کنید","Ściągnij obrazy ISO UEFI Shell","Baixar ISOs do Shell UEFI","Transferir ISOs de UEFI Shell","Descarcă UEFI Shell ISO-uri","Загрузка ISO-образов оболочки UEFI","Preuzmite UEFI Shell ISOs","Stiahnite UEFI Shell ISO","Prenos UEFI Shell ISOs","Descargar ISO de UEFI Shell","Ladda ner UEFI-skal ISO-filer","ดาวน์โหลดไฟล์ ISO ของ UEFI Shell","UEFI Shell ISO'larını indirin","Завантаження командної оболонки UEFI ISO","Tải xuống các tệp UEFI Shell ISO" "Feature14","713","Text","",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/res/loc/ChangeLog.txt b/res/loc/ChangeLog.txt index 072325df9c2..6939aa06c07 100644 --- a/res/loc/ChangeLog.txt +++ b/res/loc/ChangeLog.txt @@ -5,6 +5,10 @@ To edit a translation, please make sure to follow: https://github.com/pbatard/rufus/wiki/Localization#Editing_an_existing_translation Or simply download https://files.akeo.ie/pollock/pollock-1.5.exe and follow its directions. +o v4.? (????.??.??) ** NOT FINAL!!! PLEASE DO NOT SEND UNSOLICITED TRANSLATIONS! ** + - *NEW* MSG_350 "Use 'Windows UEFI CA 2023' signed bootloaders [EXPERIMENTAL]" + - *NEW* MSG_351 "Checking for UEFI bootloader revocation..." + o v4.5 (2024.05.??) - *UPDATED* IDC_RUFUS_MBR -> IDC_UEFI_MEDIA_VALIDATION "Enable runtime UEFI media validation" - *UPDATED* MSG_167 "Install a UEFI bootloader, that will perform MD5Sum file validation of the media" diff --git a/src/rufus.rc b/src/rufus.rc index bf9da2325c7..63419a76bce 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2207" +CAPTION "Rufus 4.6.2208" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -399,8 +399,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2207,0 - PRODUCTVERSION 4,6,2207,0 + FILEVERSION 4,6,2208,0 + PRODUCTVERSION 4,6,2208,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -418,13 +418,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2207" + VALUE "FileVersion", "4.6.2208" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", " 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.6.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2207" + VALUE "ProductVersion", "4.6.2208" END END BLOCK "VarFileInfo" From 37e383ade6eebe40c41f09f8f6b8f135118b4907 Mon Sep 17 00:00:00 2001 From: Pete Batard <pete@akeo.ie> Date: Fri, 22 Nov 2024 00:03:13 +0000 Subject: [PATCH 27/30] [dev] add disk exclusion by GPT Disk GUID * Similar to what we already do with IgnoreUSB##, except this time, users can add REG_SZ keys IgnoreDisk01 to IgnoreDisk08, with a string like "{F333EC2E-25C9-488D-A7FC-9147C2367623}" to ignore a GPT disk with this specific GUID. * This may be useful for people who mount fixed virtual drives, or people who have enabled Hot Swap on their SATA storage, and who want to make sure they won't be able to inadvertently select that disk in Rufus. * Also set rufus-next to 4.7. --- configure | 20 ++++++++++---------- configure.ac | 2 +- src/dev.c | 3 +++ src/drive.c | 42 ++++++++++++++++++++++++++++++++++++++++++ src/drive.h | 1 + src/rufus.rc | 12 ++++++------ 6 files changed, 63 insertions(+), 17 deletions(-) diff --git a/configure b/configure index 667af6b2f0b..a385d034024 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for rufus 4.6. +# Generated by GNU Autoconf 2.71 for rufus 4.7. # # Report bugs to <https://github.com/pbatard/rufus/issues>. # @@ -611,8 +611,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='rufus' PACKAGE_TARNAME='rufus' -PACKAGE_VERSION='4.6' -PACKAGE_STRING='rufus 4.6' +PACKAGE_VERSION='4.7' +PACKAGE_STRING='rufus 4.7' PACKAGE_BUGREPORT='https://github.com/pbatard/rufus/issues' PACKAGE_URL='https://rufus.ie' @@ -1269,7 +1269,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures rufus 4.6 to adapt to many kinds of systems. +\`configure' configures rufus 4.7 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1336,7 +1336,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of rufus 4.6:";; + short | recursive ) echo "Configuration of rufus 4.7:";; esac cat <<\_ACEOF @@ -1428,7 +1428,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -rufus configure 4.6 +rufus configure 4.7 generated by GNU Autoconf 2.71 Copyright (C) 2021 Free Software Foundation, Inc. @@ -1504,7 +1504,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by rufus $as_me 4.6, which was +It was created by rufus $as_me 4.7, which was generated by GNU Autoconf 2.71. Invocation command line was $ $0$ac_configure_args_raw @@ -2767,7 +2767,7 @@ fi # Define the identity of the package. PACKAGE='rufus' - VERSION='4.6' + VERSION='4.7' printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h @@ -5309,7 +5309,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by rufus $as_me 4.6, which was +This file was extended by rufus $as_me 4.7, which was generated by GNU Autoconf 2.71. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5365,7 +5365,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -rufus config.status 4.6 +rufus config.status 4.7 configured by $0, generated by GNU Autoconf 2.71, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 513ab26d6cf..c530cd1010d 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([rufus], [4.6], [https://github.com/pbatard/rufus/issues], [rufus], [https://rufus.ie]) +AC_INIT([rufus], [4.7], [https://github.com/pbatard/rufus/issues], [rufus], [https://rufus.ie]) AM_INIT_AUTOMAKE([-Wno-portability foreign no-dist no-dependencies]) AC_CONFIG_SRCDIR([src/rufus.c]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/src/dev.c b/src/dev.c index 5ea2b2a06ab..a5768a8fb90 100644 --- a/src/dev.c +++ b/src/dev.c @@ -943,6 +943,9 @@ BOOL GetDevices(DWORD devnum) uprintf("Device eliminated because it was detected as a Microsoft Dev Drive"); safe_free(devint_detail_data); break; + } else if (IsFilteredDrive(drive_index)) { + safe_free(devint_detail_data); + break; } // Windows 10 19H1 mounts a 'PortableBaseLayer' for its Windows Sandbox feature => unlist those if (safe_strcmp(label, windows_sandbox_vhd_label) == 0) { diff --git a/src/drive.c b/src/drive.c index 5f378e2424a..6be48e3a877 100644 --- a/src/drive.c +++ b/src/drive.c @@ -2647,3 +2647,45 @@ BOOL IsMsDevDrive(DWORD DriveIndex) safe_closehandle(hPhysical); return ret; } + +/* + * Detect filtered drives, that have been added by users through the registry + * entries IgnoreDisk01 - IgnoreDisk08. These entries must contain *decorated* + * string GUIDs that match the GPT Disk GUID of the drive to filter out, as + * reported by Rufus, such as "{F333EC2E-25C9-488D-A7FC-9147C2367623}". + */ +BOOL IsFilteredDrive(DWORD DriveIndex) +{ + char setting_name[32]; + DWORD i, size = 0; + BOOL r, ret = FALSE; + HANDLE hPhysical = INVALID_HANDLE_VALUE; + BYTE layout[4096] = { 0 }; + PDRIVE_LAYOUT_INFORMATION_EX DriveLayout = (PDRIVE_LAYOUT_INFORMATION_EX)(void*)layout; + GUID* DiskGuid; + + hPhysical = GetPhysicalHandle(DriveIndex, FALSE, FALSE, TRUE); + if (hPhysical == INVALID_HANDLE_VALUE) + goto out; + + r = DeviceIoControl(hPhysical, IOCTL_DISK_GET_DRIVE_LAYOUT_EX, NULL, 0, layout, sizeof(layout), &size, NULL); + if (!r || size <= 0) + goto out; + + // Only works for GPT drives + if (DriveLayout->PartitionStyle != PARTITION_STYLE_GPT) + goto out; + for (i = 1; i <= MAX_IGNORE_USB; i++) { + static_sprintf(setting_name, "IgnoreDisk%02d", i); + DiskGuid = StringToGuid(ReadSettingStr(setting_name)); + if (CompareGUID(&DriveLayout->Gpt.DiskId, DiskGuid)) { + uprintf("Device eliminated because it matches Disk GUID %s", GuidToString(DiskGuid, TRUE)); + ret = TRUE; + goto out; + } + } + +out: + safe_closehandle(hPhysical); + return ret; +} diff --git a/src/drive.h b/src/drive.h index 01e9c6b8acc..bee39f3915e 100644 --- a/src/drive.h +++ b/src/drive.h @@ -425,3 +425,4 @@ BOOL GetOpticalMedia(IMG_SAVE* img_save); uint64_t GetEspOffset(DWORD DriveIndex); BOOL ToggleEsp(DWORD DriveIndex, uint64_t PartitionOffset); BOOL IsMsDevDrive(DWORD DriveIndex); +BOOL IsFilteredDrive(DWORD DriveIndex); diff --git a/src/rufus.rc b/src/rufus.rc index 63419a76bce..6e5aacdf5ee 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.6.2208" +CAPTION "Rufus 4.7.2209" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -399,8 +399,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,2208,0 - PRODUCTVERSION 4,6,2208,0 + FILEVERSION 4,7,2209,0 + PRODUCTVERSION 4,7,2209,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -418,13 +418,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.6.2208" + VALUE "FileVersion", "4.7.2209" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", " 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" - VALUE "OriginalFilename", "rufus-4.6.exe" + VALUE "OriginalFilename", "rufus-4.7.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.6.2208" + VALUE "ProductVersion", "4.7.2209" END END BLOCK "VarFileInfo" From fcdde3dcb8db1f31d530470ad3c48f2313570705 Mon Sep 17 00:00:00 2001 From: Pete Batard <pete@akeo.ie> Date: Fri, 22 Nov 2024 00:18:58 +0000 Subject: [PATCH 28/30] [misc] drop ARM32 builds * Per https://github.com/actions/runner-images/issues/10981 and plenty of other similar reports, GitHub Actions can no longer compile ARM32 binaries by default using the latest Visual Studio toolchain, due to Microsoft current Windows SDK having dropped the ARM32 toolchain (per https://github.com/zufuliu/notepad4/issues/839). * Well, I have better things to do then try to maintain platforms in the process of being deprecated, so I'll let the people who care about Rufus on ARM32 propose a non-intrusive workaround that can work with current GitHub Actions. --- .github/workflows/vs2022.yml | 2 +- src/rufus.rc | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/vs2022.yml b/.github/workflows/vs2022.yml index 249f7ad42a4..457b2f586c7 100644 --- a/.github/workflows/vs2022.yml +++ b/.github/workflows/vs2022.yml @@ -35,7 +35,7 @@ jobs: strategy: matrix: - TARGET_PLATFORM: [x64, x86, arm64, arm] + TARGET_PLATFORM: [x64, x86, arm64] steps: - name: Checkout repository diff --git a/src/rufus.rc b/src/rufus.rc index 6e5aacdf5ee..a391c4f5de7 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.7.2209" +CAPTION "Rufus 4.7.2210" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -399,8 +399,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,7,2209,0 - PRODUCTVERSION 4,7,2209,0 + FILEVERSION 4,7,2210,0 + PRODUCTVERSION 4,7,2210,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -418,13 +418,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.7.2209" + VALUE "FileVersion", "4.7.2210" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", " 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.7.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.7.2209" + VALUE "ProductVersion", "4.7.2210" END END BLOCK "VarFileInfo" From cef7a5cb0cbcd6dd3e5341e05d0cbcecb87cbb29 Mon Sep 17 00:00:00 2001 From: Pete Batard <pete@akeo.ie> Date: Fri, 22 Nov 2024 23:24:37 +0000 Subject: [PATCH 29/30] [cmp] add ztsd image compression support * Based on the latest Bled, which adds ztsd compression support. * Note that initial extraction of the 512 bytes MBR is very slow, because is seems clear that ZSTD was never designed for fast init or processing small elements of data, but instead for post init large volume streaming. * Also note that this code adds 400 KB to the Rufus executable *AFTER UPX COMPRESSION*! Hopefully, the BusyBox folks can come up with a better and smaller way to add zstd support, because it's clear that the method used by the current BusyBox proposal, which is to leave as much of the original code untouched, isn't for the best... * Closes #2590. * Closes #2620. * Closes #2621. --- .vs/bled.vcxproj | 24 + .vs/bled.vcxproj.filters | 72 ++ src/bled/Makefile.am | 7 +- src/bled/Makefile.in | 64 +- src/bled/bb_archive.h | 7 + src/bled/bled.c | 15 +- src/bled/bled.h | 3 +- src/bled/decompress_unzstd.c | 147 +++ src/bled/fse.h | 700 +++++++++++ src/bled/fse_bitstream.h | 458 +++++++ src/bled/fse_decompress.c | 390 ++++++ src/bled/huf.h | 344 ++++++ src/bled/huf_decompress.c | 1747 +++++++++++++++++++++++++++ src/bled/libbb.h | 1 - src/bled/open_transformer.c | 12 + src/bled/xxhash.c | 821 +++++++++++++ src/bled/xxhash.h | 276 +++++ src/bled/xz_config.h | 1 - src/bled/xz_dec_lzma2.c | 1 + src/bled/zstd.h | 148 +++ src/bled/zstd_compiler.h | 242 ++++ src/bled/zstd_config.h | 49 + src/bled/zstd_cpu.h | 213 ++++ src/bled/zstd_ddict.h | 49 + src/bled/zstd_decompress.c | 1248 +++++++++++++++++++ src/bled/zstd_decompress_block.c | 1489 +++++++++++++++++++++++ src/bled/zstd_decompress_block.h | 62 + src/bled/zstd_decompress_internal.h | 205 ++++ src/bled/zstd_deps.h | 130 ++ src/bled/zstd_entropy_common.c | 362 ++++++ src/bled/zstd_error_private.c | 56 + src/bled/zstd_error_private.h | 157 +++ src/bled/zstd_errors.h | 77 ++ src/bled/zstd_internal.h | 449 +++++++ src/bled/zstd_mem.h | 423 +++++++ src/rufus.c | 2 +- src/rufus.rc | 10 +- src/vhd.c | 1 + 38 files changed, 10440 insertions(+), 22 deletions(-) create mode 100644 src/bled/decompress_unzstd.c create mode 100644 src/bled/fse.h create mode 100644 src/bled/fse_bitstream.h create mode 100644 src/bled/fse_decompress.c create mode 100644 src/bled/huf.h create mode 100644 src/bled/huf_decompress.c create mode 100644 src/bled/xxhash.c create mode 100644 src/bled/xxhash.h create mode 100644 src/bled/zstd.h create mode 100644 src/bled/zstd_compiler.h create mode 100644 src/bled/zstd_config.h create mode 100644 src/bled/zstd_cpu.h create mode 100644 src/bled/zstd_ddict.h create mode 100644 src/bled/zstd_decompress.c create mode 100644 src/bled/zstd_decompress_block.c create mode 100644 src/bled/zstd_decompress_block.h create mode 100644 src/bled/zstd_decompress_internal.h create mode 100644 src/bled/zstd_deps.h create mode 100644 src/bled/zstd_entropy_common.c create mode 100644 src/bled/zstd_error_private.c create mode 100644 src/bled/zstd_error_private.h create mode 100644 src/bled/zstd_errors.h create mode 100644 src/bled/zstd_internal.h create mode 100644 src/bled/zstd_mem.h diff --git a/.vs/bled.vcxproj b/.vs/bled.vcxproj index 94e9e48e3ca..2c25bf54449 100644 --- a/.vs/bled.vcxproj +++ b/.vs/bled.vcxproj @@ -46,32 +46,56 @@ <ClCompile Include="..\src\bled\decompress_unlzma.c" /> <ClCompile Include="..\src\bled\decompress_unxz.c" /> <ClCompile Include="..\src\bled\decompress_unzip.c" /> + <ClCompile Include="..\src\bled\decompress_unzstd.c" /> <ClCompile Include="..\src\bled\decompress_vtsi.c" /> <ClCompile Include="..\src\bled\filter_accept_all.c" /> <ClCompile Include="..\src\bled\filter_accept_list.c" /> <ClCompile Include="..\src\bled\filter_accept_reject_list.c" /> <ClCompile Include="..\src\bled\find_list_entry.c" /> + <ClCompile Include="..\src\bled\fse_decompress.c" /> <ClCompile Include="..\src\bled\header_list.c" /> <ClCompile Include="..\src\bled\header_skip.c" /> <ClCompile Include="..\src\bled\header_verbose_list.c" /> + <ClCompile Include="..\src\bled\huf_decompress.c" /> <ClCompile Include="..\src\bled\init_handle.c" /> <ClCompile Include="..\src\bled\open_transformer.c" /> <ClCompile Include="..\src\bled\seek_by_jump.c" /> <ClCompile Include="..\src\bled\seek_by_read.c" /> + <ClCompile Include="..\src\bled\xxhash.c" /> <ClCompile Include="..\src\bled\xz_dec_bcj.c" /> <ClCompile Include="..\src\bled\xz_dec_lzma2.c" /> <ClCompile Include="..\src\bled\xz_dec_stream.c" /> + <ClCompile Include="..\src\bled\zstd_decompress.c" /> + <ClCompile Include="..\src\bled\zstd_decompress_block.c" /> + <ClCompile Include="..\src\bled\zstd_entropy_common.c" /> + <ClCompile Include="..\src\bled\zstd_error_private.c" /> </ItemGroup> <ItemGroup> <ClInclude Include="..\src\bled\bb_archive.h" /> <ClInclude Include="..\src\bled\bled.h" /> + <ClInclude Include="..\src\bled\fse.h" /> + <ClInclude Include="..\src\bled\fse_bitstream.h" /> + <ClInclude Include="..\src\bled\huf.h" /> <ClInclude Include="..\src\bled\libbb.h" /> <ClInclude Include="..\src\bled\platform.h" /> + <ClInclude Include="..\src\bled\xxhash.h" /> <ClInclude Include="..\src\bled\xz.h" /> <ClInclude Include="..\src\bled\xz_config.h" /> <ClInclude Include="..\src\bled\xz_lzma2.h" /> <ClInclude Include="..\src\bled\xz_private.h" /> <ClInclude Include="..\src\bled\xz_stream.h" /> + <ClInclude Include="..\src\bled\zstd.h" /> + <ClInclude Include="..\src\bled\zstd_compiler.h" /> + <ClInclude Include="..\src\bled\zstd_config.h" /> + <ClInclude Include="..\src\bled\zstd_cpu.h" /> + <ClInclude Include="..\src\bled\zstd_ddict.h" /> + <ClInclude Include="..\src\bled\zstd_decompress_block.h" /> + <ClInclude Include="..\src\bled\zstd_decompress_internal.h" /> + <ClInclude Include="..\src\bled\zstd_deps.h" /> + <ClInclude Include="..\src\bled\zstd_errors.h" /> + <ClInclude Include="..\src\bled\zstd_error_private.h" /> + <ClInclude Include="..\src\bled\zstd_internal.h" /> + <ClInclude Include="..\src\bled\zstd_mem.h" /> </ItemGroup> <PropertyGroup Label="Globals"> <ProjectName>bled</ProjectName> diff --git a/.vs/bled.vcxproj.filters b/.vs/bled.vcxproj.filters index f7370b6bb29..aec232074bc 100644 --- a/.vs/bled.vcxproj.filters +++ b/.vs/bled.vcxproj.filters @@ -87,6 +87,30 @@ <ClCompile Include="..\src\bled\decompress_vtsi.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\src\bled\decompress_unzstd.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\src\bled\fse_decompress.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\src\bled\huf_decompress.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\src\bled\xxhash.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\src\bled\zstd_decompress.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\src\bled\zstd_decompress_block.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\src\bled\zstd_entropy_common.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\src\bled\zstd_error_private.c"> + <Filter>Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <ClInclude Include="..\src\bled\bb_archive.h"> @@ -116,5 +140,53 @@ <ClInclude Include="..\src\bled\xz_stream.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\src\bled\fse.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\fse_bitstream.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\huf.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\xxhash.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\zstd.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\zstd_compiler.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\zstd_config.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\zstd_cpu.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\zstd_ddict.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\zstd_decompress_block.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\zstd_decompress_internal.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\zstd_deps.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\zstd_error_private.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\zstd_errors.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\zstd_internal.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\src\bled\zstd_mem.h"> + <Filter>Header Files</Filter> + </ClInclude> </ItemGroup> </Project> \ No newline at end of file diff --git a/src/bled/Makefile.am b/src/bled/Makefile.am index c4e7d8004ab..07417ca20d9 100644 --- a/src/bled/Makefile.am +++ b/src/bled/Makefile.am @@ -2,7 +2,8 @@ noinst_LIBRARIES = libbled.a libbled_a_SOURCES = bled.c crc32.c data_align.c data_extract_all.c data_skip.c decompress_bunzip2.c \ decompress_gunzip.c decompress_uncompress.c decompress_unlzma.c decompress_unxz.c decompress_unzip.c \ - decompress_vtsi.c filter_accept_all.c filter_accept_list.c filter_accept_reject_list.c find_list_entry.c \ - header_list.c header_skip.c header_verbose_list.c init_handle.c open_transformer.c \ - seek_by_jump.c seek_by_read.c xz_dec_bcj.c xz_dec_lzma2.c xz_dec_stream.c + decompress_unzstd.c decompress_vtsi.c filter_accept_all.c filter_accept_list.c filter_accept_reject_list.c \ + find_list_entry.c fse_decompress.c header_list.c header_skip.c header_verbose_list.c huf_decompress.c \ + init_handle.c open_transformer.c seek_by_jump.c seek_by_read.c xz_dec_bcj.c xz_dec_lzma2.c xz_dec_stream.c \ + xxhash.c zstd_decompress.c zstd_decompress_block.c zstd_entropy_common.c zstd_error_private.c libbled_a_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/.. -Wno-undef -Wno-strict-aliasing diff --git a/src/bled/Makefile.in b/src/bled/Makefile.in index 7ca170f8a70..cc7463e3c46 100644 --- a/src/bled/Makefile.in +++ b/src/bled/Makefile.in @@ -103,21 +103,28 @@ am_libbled_a_OBJECTS = libbled_a-bled.$(OBJEXT) \ libbled_a-decompress_unlzma.$(OBJEXT) \ libbled_a-decompress_unxz.$(OBJEXT) \ libbled_a-decompress_unzip.$(OBJEXT) \ + libbled_a-decompress_unzstd.$(OBJEXT) \ libbled_a-decompress_vtsi.$(OBJEXT) \ libbled_a-filter_accept_all.$(OBJEXT) \ libbled_a-filter_accept_list.$(OBJEXT) \ libbled_a-filter_accept_reject_list.$(OBJEXT) \ libbled_a-find_list_entry.$(OBJEXT) \ + libbled_a-fse_decompress.$(OBJEXT) \ libbled_a-header_list.$(OBJEXT) \ libbled_a-header_skip.$(OBJEXT) \ libbled_a-header_verbose_list.$(OBJEXT) \ + libbled_a-huf_decompress.$(OBJEXT) \ libbled_a-init_handle.$(OBJEXT) \ libbled_a-open_transformer.$(OBJEXT) \ libbled_a-seek_by_jump.$(OBJEXT) \ libbled_a-seek_by_read.$(OBJEXT) \ libbled_a-xz_dec_bcj.$(OBJEXT) \ libbled_a-xz_dec_lzma2.$(OBJEXT) \ - libbled_a-xz_dec_stream.$(OBJEXT) + libbled_a-xz_dec_stream.$(OBJEXT) libbled_a-xxhash.$(OBJEXT) \ + libbled_a-zstd_decompress.$(OBJEXT) \ + libbled_a-zstd_decompress_block.$(OBJEXT) \ + libbled_a-zstd_entropy_common.$(OBJEXT) \ + libbled_a-zstd_error_private.$(OBJEXT) libbled_a_OBJECTS = $(am_libbled_a_OBJECTS) AM_V_P = $(am__v_P_@AM_V@) am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) @@ -269,9 +276,10 @@ top_srcdir = @top_srcdir@ noinst_LIBRARIES = libbled.a libbled_a_SOURCES = bled.c crc32.c data_align.c data_extract_all.c data_skip.c decompress_bunzip2.c \ decompress_gunzip.c decompress_uncompress.c decompress_unlzma.c decompress_unxz.c decompress_unzip.c \ - decompress_vtsi.c filter_accept_all.c filter_accept_list.c filter_accept_reject_list.c find_list_entry.c \ - header_list.c header_skip.c header_verbose_list.c init_handle.c open_transformer.c \ - seek_by_jump.c seek_by_read.c xz_dec_bcj.c xz_dec_lzma2.c xz_dec_stream.c + decompress_unzstd.c decompress_vtsi.c filter_accept_all.c filter_accept_list.c filter_accept_reject_list.c \ + find_list_entry.c fse_decompress.c header_list.c header_skip.c header_verbose_list.c huf_decompress.c \ + init_handle.c open_transformer.c seek_by_jump.c seek_by_read.c xz_dec_bcj.c xz_dec_lzma2.c xz_dec_stream.c \ + xxhash.c zstd_decompress.c zstd_decompress_block.c zstd_entropy_common.c zstd_error_private.c libbled_a_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/.. -Wno-undef -Wno-strict-aliasing all: all-am @@ -395,6 +403,12 @@ libbled_a-decompress_unzip.o: decompress_unzip.c libbled_a-decompress_unzip.obj: decompress_unzip.c $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-decompress_unzip.obj `if test -f 'decompress_unzip.c'; then $(CYGPATH_W) 'decompress_unzip.c'; else $(CYGPATH_W) '$(srcdir)/decompress_unzip.c'; fi` +libbled_a-decompress_unzstd.o: decompress_unzstd.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-decompress_unzstd.o `test -f 'decompress_unzstd.c' || echo '$(srcdir)/'`decompress_unzstd.c + +libbled_a-decompress_unzstd.obj: decompress_unzstd.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-decompress_unzstd.obj `if test -f 'decompress_unzstd.c'; then $(CYGPATH_W) 'decompress_unzstd.c'; else $(CYGPATH_W) '$(srcdir)/decompress_unzstd.c'; fi` + libbled_a-decompress_vtsi.o: decompress_vtsi.c $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-decompress_vtsi.o `test -f 'decompress_vtsi.c' || echo '$(srcdir)/'`decompress_vtsi.c @@ -425,6 +439,12 @@ libbled_a-find_list_entry.o: find_list_entry.c libbled_a-find_list_entry.obj: find_list_entry.c $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-find_list_entry.obj `if test -f 'find_list_entry.c'; then $(CYGPATH_W) 'find_list_entry.c'; else $(CYGPATH_W) '$(srcdir)/find_list_entry.c'; fi` +libbled_a-fse_decompress.o: fse_decompress.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-fse_decompress.o `test -f 'fse_decompress.c' || echo '$(srcdir)/'`fse_decompress.c + +libbled_a-fse_decompress.obj: fse_decompress.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-fse_decompress.obj `if test -f 'fse_decompress.c'; then $(CYGPATH_W) 'fse_decompress.c'; else $(CYGPATH_W) '$(srcdir)/fse_decompress.c'; fi` + libbled_a-header_list.o: header_list.c $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-header_list.o `test -f 'header_list.c' || echo '$(srcdir)/'`header_list.c @@ -443,6 +463,12 @@ libbled_a-header_verbose_list.o: header_verbose_list.c libbled_a-header_verbose_list.obj: header_verbose_list.c $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-header_verbose_list.obj `if test -f 'header_verbose_list.c'; then $(CYGPATH_W) 'header_verbose_list.c'; else $(CYGPATH_W) '$(srcdir)/header_verbose_list.c'; fi` +libbled_a-huf_decompress.o: huf_decompress.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-huf_decompress.o `test -f 'huf_decompress.c' || echo '$(srcdir)/'`huf_decompress.c + +libbled_a-huf_decompress.obj: huf_decompress.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-huf_decompress.obj `if test -f 'huf_decompress.c'; then $(CYGPATH_W) 'huf_decompress.c'; else $(CYGPATH_W) '$(srcdir)/huf_decompress.c'; fi` + libbled_a-init_handle.o: init_handle.c $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-init_handle.o `test -f 'init_handle.c' || echo '$(srcdir)/'`init_handle.c @@ -485,6 +511,36 @@ libbled_a-xz_dec_stream.o: xz_dec_stream.c libbled_a-xz_dec_stream.obj: xz_dec_stream.c $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-xz_dec_stream.obj `if test -f 'xz_dec_stream.c'; then $(CYGPATH_W) 'xz_dec_stream.c'; else $(CYGPATH_W) '$(srcdir)/xz_dec_stream.c'; fi` +libbled_a-xxhash.o: xxhash.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-xxhash.o `test -f 'xxhash.c' || echo '$(srcdir)/'`xxhash.c + +libbled_a-xxhash.obj: xxhash.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-xxhash.obj `if test -f 'xxhash.c'; then $(CYGPATH_W) 'xxhash.c'; else $(CYGPATH_W) '$(srcdir)/xxhash.c'; fi` + +libbled_a-zstd_decompress.o: zstd_decompress.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_decompress.o `test -f 'zstd_decompress.c' || echo '$(srcdir)/'`zstd_decompress.c + +libbled_a-zstd_decompress.obj: zstd_decompress.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_decompress.obj `if test -f 'zstd_decompress.c'; then $(CYGPATH_W) 'zstd_decompress.c'; else $(CYGPATH_W) '$(srcdir)/zstd_decompress.c'; fi` + +libbled_a-zstd_decompress_block.o: zstd_decompress_block.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_decompress_block.o `test -f 'zstd_decompress_block.c' || echo '$(srcdir)/'`zstd_decompress_block.c + +libbled_a-zstd_decompress_block.obj: zstd_decompress_block.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_decompress_block.obj `if test -f 'zstd_decompress_block.c'; then $(CYGPATH_W) 'zstd_decompress_block.c'; else $(CYGPATH_W) '$(srcdir)/zstd_decompress_block.c'; fi` + +libbled_a-zstd_entropy_common.o: zstd_entropy_common.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_entropy_common.o `test -f 'zstd_entropy_common.c' || echo '$(srcdir)/'`zstd_entropy_common.c + +libbled_a-zstd_entropy_common.obj: zstd_entropy_common.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_entropy_common.obj `if test -f 'zstd_entropy_common.c'; then $(CYGPATH_W) 'zstd_entropy_common.c'; else $(CYGPATH_W) '$(srcdir)/zstd_entropy_common.c'; fi` + +libbled_a-zstd_error_private.o: zstd_error_private.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_error_private.o `test -f 'zstd_error_private.c' || echo '$(srcdir)/'`zstd_error_private.c + +libbled_a-zstd_error_private.obj: zstd_error_private.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_error_private.obj `if test -f 'zstd_error_private.c'; then $(CYGPATH_W) 'zstd_error_private.c'; else $(CYGPATH_W) '$(srcdir)/zstd_error_private.c'; fi` + ID: $(am__tagged_files) $(am__define_uniq_tagged_files); mkid -fID $$unique tags: tags-am diff --git a/src/bled/bb_archive.h b/src/bled/bb_archive.h index 5e059a5ad31..36fca55c285 100644 --- a/src/bled/bb_archive.h +++ b/src/bled/bb_archive.h @@ -17,6 +17,9 @@ enum { /* (unsigned) cast suppresses "integer overflow in expression" warning */ XZ_MAGIC1a = 256 * (unsigned)(256 * (256 * 0xfd + '7') + 'z') + 'X', XZ_MAGIC2a = 256 * 'Z' + 0, + ZSTD_MAGIC1 = 0x28B5, + ZSTD_MAGIC2 = 0x2FFD, + ZSTD_MAGIC = 0x28B52FFD, #else COMPRESS_MAGIC = 0x9d1f, GZIP_MAGIC = 0x8b1f, @@ -25,6 +28,9 @@ enum { XZ_MAGIC2 = 'z' + ('X' + ('Z' + 0 * 256) * 256) * 256, XZ_MAGIC1a = 0xfd + ('7' + ('z' + 'X' * 256) * 256) * 256, XZ_MAGIC2a = 'Z' + 0 * 256, + ZSTD_MAGIC1 = 0xB528, + ZSTD_MAGIC2 = 0xFD2F, + ZSTD_MAGIC = 0xFD2FB528, #endif }; @@ -290,6 +296,7 @@ IF_DESKTOP(long long) int unpack_bz2_stream(transformer_state_t *xstate) FAST_FU IF_DESKTOP(long long) int unpack_lzma_stream(transformer_state_t *xstate) FAST_FUNC; IF_DESKTOP(long long) int unpack_xz_stream(transformer_state_t *xstate) FAST_FUNC; IF_DESKTOP(long long) int unpack_vtsi_stream(transformer_state_t *xstate) FAST_FUNC; +IF_DESKTOP(long long) int unpack_zstd_stream(transformer_state_t *xstate) FAST_FUNC; char* append_ext(char *filename, const char *expected_ext) FAST_FUNC; int bbunpack(char **argv, diff --git a/src/bled/bled.c b/src/bled/bled.c index 7d1c813cee7..f73508e1aaf 100644 --- a/src/bled/bled.c +++ b/src/bled/bled.c @@ -1,7 +1,7 @@ /* * Bled (Base Library for Easy Decompression) * - * Copyright © 2014-2023 Pete Batard <pete@akeo.ie> + * Copyright © 2014-2024 Pete Batard <pete@akeo.ie> * * Licensed under GPLv2 or later, see file LICENSE in this source tree. */ @@ -31,7 +31,9 @@ jmp_buf bb_error_jmp; char* bb_virtual_buf = NULL; size_t bb_virtual_len = 0, bb_virtual_pos = 0; int bb_virtual_fd = -1; -uint32_t BB_BUFSIZE = 0x10000; +// ZSTD has a minimal buffer size of (1 << ZSTD_BLOCKSIZELOG_MAX) + ZSTD_blockHeaderSize = 128 KB + 3 +// So we set our bufsize to 256 KB +uint32_t BB_BUFSIZE = 0x40000; static long long int unpack_none(transformer_state_t *xstate) { @@ -49,6 +51,7 @@ unpacker_t unpacker[BLED_COMPRESSION_MAX] = { unpack_xz_stream, unpack_none, unpack_vtsi_stream, + unpack_zstd_stream, }; /* Uncompress file 'src', compressed using 'type', to file 'dst' */ @@ -267,7 +270,7 @@ int64_t bled_uncompress_from_buffer_to_buffer(const char* src, const size_t src_ /* Initialize the library. * When the parameters are not NULL or zero you can: - * - specify the buffer size to use (must be larger than 64KB and a power of two) + * - specify the buffer size to use (must be larger than 256KB and a power of two) * - specify the printf-like function you want to use to output message * void print_function(const char* format, ...); * - specify the read/write functions you want to use; @@ -283,11 +286,11 @@ int bled_init(uint32_t buffer_size, printf_t print_function, read_t read_functio if (bled_initialized) return -1; BB_BUFSIZE = buffer_size; - /* buffer_size must be larger than 64 KB and a power of two */ - if (buffer_size < 0x10000 || (buffer_size & (buffer_size - 1)) != 0) { + /* buffer_size must be larger than 256 KB and a power of two */ + if (buffer_size < 0x40000 || (buffer_size & (buffer_size - 1)) != 0) { if (buffer_size != 0 && print_function != NULL) print_function("bled_init: invalid buffer_size, defaulting to 64 KB"); - BB_BUFSIZE = 0x10000; + BB_BUFSIZE = 0x40000; } bled_printf = print_function; bled_read = read_function; diff --git a/src/bled/bled.h b/src/bled/bled.h index 480498cf69e..7a354cad3fd 100644 --- a/src/bled/bled.h +++ b/src/bled/bled.h @@ -1,7 +1,7 @@ /* * Bled (Base Library for Easy Decompression) * - * Copyright © 2014-2015 Pete Batard <pete@akeo.ie> + * Copyright © 2014-2024 Pete Batard <pete@akeo.ie> * * Licensed under GPLv2 or later, see file LICENSE in this source tree. */ @@ -31,6 +31,7 @@ typedef enum { BLED_COMPRESSION_XZ, // .xz BLED_COMPRESSION_7ZIP, // .7z BLED_COMPRESSION_VTSI, // .vtsi + BLED_COMPRESSION_ZSTD, // .zst BLED_COMPRESSION_MAX } bled_compression_type; diff --git a/src/bled/decompress_unzstd.c b/src/bled/decompress_unzstd.c new file mode 100644 index 00000000000..1792571fdd6 --- /dev/null +++ b/src/bled/decompress_unzstd.c @@ -0,0 +1,147 @@ +/* vi: set sw=4 ts=4: */ +/* + * Glue for zstd decompression + * Copyright (c) 2021 Norbert Lange <nolange79@gmail.com> + * Copyright (c) 2024 Pete Batard <pete@akeo.ie> + * + * Based on compress.c from the systemd project, + * provided by Norbert Lange <nolange79@gmail.com>. + * Which originally was copied from the streaming_decompression.c + * example from the zstd project, written by Yann Collet + * + * Licensed under GPLv2 or later, see file LICENSE in this source tree. + */ + +#include "libbb.h" +#include "bb_archive.h" +#include "zstd_deps.h" +#include "zstd_internal.h" + +ZSTD_customMem ZSTD_defaultCMem = { ZSTD_customMalloc, ZSTD_customFree, NULL }; + +ZSTDLIB_API const char* ZSTD_getErrorName(size_t code) { + return ERR_getErrorName(code); +} + +ALWAYS_INLINE static size_t roundupsize(size_t size, size_t align) +{ + return (size + align - 1U) & ~(align - 1); +} + +ALWAYS_INLINE static IF_DESKTOP(long long) int +unpack_zstd_stream_inner(transformer_state_t *xstate, + ZSTD_DStream *dctx, void *out_buff) +{ + const U32 zstd_magic = ZSTD_MAGIC; + const size_t in_allocsize = roundupsize(ZSTD_DStreamInSize(), 1024), + out_allocsize = roundupsize(ZSTD_DStreamOutSize(), 1024); + + IF_DESKTOP(long long int total = 0;) + size_t last_result = ZSTD_error_maxCode + 1; + ssize_t nwrote = 0; + unsigned input_fixup; + void *in_buff = (char *)out_buff + out_allocsize; + + memcpy(in_buff, &zstd_magic, 4); + input_fixup = xstate->signature_skipped ? 4 : 0; + + /* This loop assumes that the input file is one or more concatenated + * zstd streams. This example won't work if there is trailing non-zstd + * data at the end, but streaming decompression in general handles this + * case. ZSTD_decompressStream() returns 0 exactly when the frame is + * completed, and doesn't consume input after the frame. + */ + for (;;) { + bool has_error = false; + ZSTD_inBuffer input; + ssize_t red; + + red = safe_read(xstate->src_fd, (char *)in_buff + input_fixup, (unsigned int)(in_allocsize - input_fixup)); + if (red < 0) { + bb_perror_msg(bb_msg_read_error); + return -1; + } + if (red == 0) { + break; + } + + input.src = in_buff; + input.size = (size_t)red + input_fixup; + input.pos = 0; + input_fixup = 0; + + /* Given a valid frame, zstd won't consume the last byte of the + * frame until it has flushed all of the decompressed data of + * the frame. So input.pos < input.size means frame is not done + * or there is still output available. + */ + while (input.pos < input.size) { + ZSTD_outBuffer output = { out_buff, out_allocsize, 0 }; + /* The return code is zero if the frame is complete, but + * there may be multiple frames concatenated together. + * Zstd will automatically reset the context when a + * frame is complete. Still, calling ZSTD_DCtx_reset() + * can be useful to reset the context to a clean state, + * for instance if the last decompression call returned + * an error. + */ + last_result = ZSTD_decompressStream(dctx, &output, &input); + if (ZSTD_isError(last_result)) { + has_error = true; + break; + } + + nwrote = transformer_write(xstate, output.dst, output.pos); + if (nwrote < 0 && nwrote != -ENOSPC) { + has_error = true; + break; + } + IF_DESKTOP(total = (nwrote == -ENOSPC) ? xstate->mem_output_size_max : total + output.pos); + } + if (has_error) + break; + } + + if (last_result != 0) { + /* The last return value from ZSTD_decompressStream did not end + * on a frame, but we reached the end of the file! We assume + * this is an error, and the input was truncated. + */ + if (last_result == ZSTD_error_maxCode + 1) { + bb_simple_error_msg("could not read zstd data"); + } else { +#if defined(ZSTD_STRIP_ERROR_STRINGS) && ZSTD_STRIP_ERROR_STRINGS == 1 + bb_error_msg("zstd decoder error: %u", (unsigned)last_result); +#else + bb_error_msg("zstd decoder error: %s", ZSTD_getErrorName(last_result)); +#endif + } + return -1; + } + + return IF_DESKTOP(total) + 0; +} + +IF_DESKTOP(long long) int FAST_FUNC +unpack_zstd_stream(transformer_state_t *xstate) +{ + const size_t in_allocsize = roundupsize(ZSTD_DStreamInSize(), 1024), + out_allocsize = roundupsize(ZSTD_DStreamOutSize(), 1024); + + IF_DESKTOP(long long) int result; + void *out_buff; + ZSTD_DStream *dctx; + + dctx = ZSTD_createDStream(); + if (!dctx) { + /* should be the only possibly reason of failure */ + bb_error_msg_and_die("memory exhausted"); + } + + out_buff = xmalloc(in_allocsize + out_allocsize); + + result = unpack_zstd_stream_inner(xstate, dctx, out_buff); + free(out_buff); + ZSTD_freeDStream(dctx); + return result; +} diff --git a/src/bled/fse.h b/src/bled/fse.h new file mode 100644 index 00000000000..9bf7cb43cc6 --- /dev/null +++ b/src/bled/fse.h @@ -0,0 +1,700 @@ +/* ****************************************************************** + * FSE : Finite State Entropy codec + * Public Prototypes declaration + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +#ifndef FSE_H +#define FSE_H + + +/*-***************************************** +* Dependencies +******************************************/ +#include "zstd_deps.h" /* size_t, ptrdiff_t */ + + +# define FSE_PUBLIC_API MEM_STATIC + +/*------ Version ------*/ +#define FSE_VERSION_MAJOR 0 +#define FSE_VERSION_MINOR 9 +#define FSE_VERSION_RELEASE 0 + +#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE +#define FSE_QUOTE(str) #str +#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) +#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) + +#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE) +FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ + + +/*-**************************************** +* FSE simple functions +******************************************/ +/*! FSE_compress() : + Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. + 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). + @return : size of compressed data (<= dstCapacity). + Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. + if FSE_isError(return), compression failed (more details using FSE_getErrorName()) +*/ +FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/*! FSE_decompress(): + Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', + into already allocated destination buffer 'dst', of size 'dstCapacity'. + @return : size of regenerated data (<= maxDstSize), + or an error code, which can be tested using FSE_isError() . + + ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! + Why ? : making this distinction requires a header. + Header management is intentionally delegated to the user layer, which can better manage special cases. +*/ +FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize); + + +/*-***************************************** +* Tool functions +******************************************/ +FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ + +/* Error Management */ +unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + + +/*-***************************************** +* FSE advanced functions +******************************************/ +/*! FSE_compress2() : + Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' + Both parameters can be defined as '0' to mean : use default value + @return : size of compressed data + Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. + if FSE_isError(return), it's an error code. +*/ +FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + + +/*-***************************************** +* FSE detailed API +******************************************/ +/*! +FSE_compress() does the following: +1. count symbol occurrence from source[] into table count[] (see hist.h) +2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) +3. save normalized counters to memory buffer using writeNCount() +4. build encoding table 'CTable' from normalized counters +5. encode the data stream using encoding table 'CTable' + +FSE_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + +/* *** COMPRESSION *** */ + +/*! FSE_optimalTableLog(): + dynamically downsize 'tableLog' when conditions are met. + It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. + @return : recommended tableLog (necessarily <= 'maxTableLog') */ +FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_normalizeCount(): + normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) + 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + useLowProbCount is a boolean parameter which trades off compressed size for + faster header decoding. When it is set to 1, the compressed data will be slightly + smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be + faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0 + is a good default, since header deserialization makes a big speed difference. + Otherwise, useLowProbCount=1 is a good default, since the speed difference is small. + @return : tableLog, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount); + +/*! FSE_NCountWriteBound(): + Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. + Typically useful for allocation purpose. */ +FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_writeNCount(): + Compactly save 'normalizedCounter' into 'buffer'. + @return : size of the compressed table, + or an errorCode, which can be tested using FSE_isError(). */ +FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, + unsigned maxSymbolValue, unsigned tableLog); + +/*! Constructor and Destructor of FSE_CTable. + Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ +FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog); +FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct); + +/*! FSE_buildCTable(): + Builds `ct`, which must be already allocated, using FSE_createCTable(). + @return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_compress_usingCTable(): + Compress `src` using `ct` into `dst` which must be already allocated. + @return : size of compressed data (<= `dstCapacity`), + or 0 if compressed data could not fit into `dst`, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); + +/*! +Tutorial : +---------- +The first step is to count all symbols. FSE_count() does this job very fast. +Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. +'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] +maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) +FSE_count() will return the number of occurrence of the most frequent symbol. +This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). + +The next step is to normalize the frequencies. +FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. +It also guarantees a minimum of 1 to any Symbol with frequency >= 1. +You can use 'tableLog'==0 to mean "use default tableLog value". +If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), +which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). + +The result of FSE_normalizeCount() will be saved into a table, +called 'normalizedCounter', which is a table of signed short. +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. +The return value is tableLog if everything proceeded as expected. +It is 0 if there is a single symbol within distribution. +If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). + +'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). +'buffer' must be already allocated. +For guaranteed success, buffer size must be at least FSE_headerBound(). +The result of the function is the number of bytes written into 'buffer'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). + +'normalizedCounter' can then be used to create the compression table 'CTable'. +The space required by 'CTable' must be already allocated, using FSE_createCTable(). +You can then use FSE_buildCTable() to fill 'CTable'. +If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). + +'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). +Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' +The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. +If it returns '0', compressed data could not fit into 'dst'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). +*/ + + +/* *** DECOMPRESSION *** */ + +/*! FSE_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSE_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +size_t FSE_readNCount (short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize); + +/*! FSE_readNCount_bmi2(): + * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise. + */ +size_t FSE_readNCount_bmi2(short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize, int bmi2); + +/*! Constructor and Destructor of FSE_DTable. + Note that its size depends on 'tableLog' */ +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ +FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); +FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); + +/*! FSE_buildDTable(): + Builds 'dt', which must be already allocated, using FSE_createDTable(). + return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_decompress_usingDTable(): + Decompress compressed source `cSrc` of size `cSrcSize` using `dt` + into `dst` which must be already allocated. + @return : size of regenerated data (necessarily <= `dstCapacity`), + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. +This is performed by the function FSE_buildDTable(). +The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) +*/ + +#endif /* FSE_H */ + +#if !defined(FSE_H_FSE_STATIC_LINKING_ONLY) +#define FSE_H_FSE_STATIC_LINKING_ONLY + +/* *** Dependency *** */ +#include "fse_bitstream.h" + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */) +#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog))) + +/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */ +#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable)) +#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable)) + + +/* ***************************************** + * FSE advanced API + ***************************************** */ + +MEM_STATIC unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSE_optimalTableLog(), which used `minus==2` */ + +/* FSE_compress_wksp() : + * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). + * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. + */ +#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) +MEM_STATIC size_t FSE_compress_wksp(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +MEM_STATIC size_t FSE_buildCTable_raw(FSE_CTable* ct, unsigned nbBits); +/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ + +MEM_STATIC size_t FSE_buildCTable_rle(FSE_CTable* ct, unsigned char symbolValue); +/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`. + * See FSE_buildCTable_wksp() for breakdown of workspace usage. + */ +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */) +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)) +MEM_STATIC size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8) +#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned)) +FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); +/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ + +MEM_STATIC size_t FSE_buildDTable_raw(FSE_DTable* dt, unsigned nbBits); +/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */ + +MEM_STATIC size_t FSE_buildDTable_rle(FSE_DTable* dt, unsigned char symbolValue); +/**< build a fake FSE_DTable, designed to always generate the same symbolValue */ + +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) +#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) +MEM_STATIC size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize); +/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */ + +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); +/**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */ + +typedef enum { + FSE_repeat_none, /**< Cannot use the previous table */ + FSE_repeat_check, /**< Can use the previous table but it must be checked */ + FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */ + } FSE_repeat; + +/* ***************************************** +* FSE symbol compression API +*******************************************/ +/*! + This API consists of small unitary functions, which highly benefit from being inlined. + Hence their body are included in next section. +*/ +typedef struct { + ptrdiff_t value; + const void* stateTable; + const void* symbolTT; + unsigned stateLog; +} FSE_CState_t; + +static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct); + +static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol); + +static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr); + +/**< +These functions are inner components of FSE_compress_usingCTable(). +They allow the creation of custom streams, mixing multiple tables and bit sources. + +A key property to keep in mind is that encoding and decoding are done **in reverse direction**. +So the first symbol you will encode is the last you will decode, like a LIFO stack. + +You will need a few variables to track your CStream. They are : + +FSE_CTable ct; // Provided by FSE_buildCTable() +BIT_CStream_t bitStream; // bitStream tracking structure +FSE_CState_t state; // State tracking structure (can have several) + + +The first thing to do is to init bitStream and state. + size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize); + FSE_initCState(&state, ct); + +Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError(); +You can then encode your input data, byte after byte. +FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time. +Remember decoding will be done in reverse direction. + FSE_encodeByte(&bitStream, &state, symbol); + +At any time, you can also add any bit sequence. +Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders + BIT_addBits(&bitStream, bitField, nbBits); + +The above methods don't commit data to memory, they just store it into local register, for speed. +Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +Writing data to memory is a manual operation, performed by the flushBits function. + BIT_flushBits(&bitStream); + +Your last FSE encoding operation shall be to flush your last state value(s). + FSE_flushState(&bitStream, &state); + +Finally, you must close the bitStream. +The function returns the size of CStream in bytes. +If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible) +If there is an error, it returns an errorCode (which can be tested using FSE_isError()). + size_t size = BIT_closeCStream(&bitStream); +*/ + + +/* ***************************************** +* FSE symbol decompression API +*******************************************/ +typedef struct { + size_t state; + const void* table; /* precise table may vary, depending on U16 */ +} FSE_DState_t; + + +static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt); + +static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); + +static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr); + +/**< +Let's now decompose FSE_decompress_usingDTable() into its unitary components. +You will decode FSE-encoded symbols from the bitStream, +and also any other bitFields you put in, **in reverse order**. + +You will need a few variables to track your bitStream. They are : + +BIT_DStream_t DStream; // Stream context +FSE_DState_t DState; // State context. Multiple ones are possible +FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable() + +The first thing to do is to init the bitStream. + errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize); + +You should then retrieve your initial state(s) +(in reverse flushing order if you have several ones) : + errorCode = FSE_initDState(&DState, &DStream, DTablePtr); + +You can then decode your data, symbol after symbol. +For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'. +Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). + unsigned char symbol = FSE_decodeSymbol(&DState, &DStream); + +You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) +Note : maximum allowed nbBits is 25, for 32-bits compatibility + size_t bitField = BIT_readBits(&DStream, nbBits); + +All above operations only read from local register (which size depends on size_t). +Refueling the register from memory is manually performed by the reload method. + endSignal = FSE_reloadDStream(&DStream); + +BIT_reloadDStream() result tells if there is still some more data to read from DStream. +BIT_DStream_unfinished : there is still some data left into the DStream. +BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. +BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. +BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted. + +When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, +to properly detect the exact end of stream. +After each decoded symbol, check if DStream is fully consumed using this simple test : + BIT_reloadDStream(&DStream) >= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/* ***************************************** +* Implementation of inlined functions +*******************************************/ +typedef struct { + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) +{ + const void* ptr = ct; + const U16* u16ptr = (const U16*) ptr; + const U32 tableLog = MEM_read16(ptr); + statePtr->value = (ptrdiff_t)1<<tableLog; + statePtr->stateTable = u16ptr+2; + statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1); + statePtr->stateLog = tableLog; +} + + +/*! FSE_initCState2() : +* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) +* uses the smallest state value possible, saving the cost of this symbol */ +MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) +{ + FSE_initCState(statePtr, ct); + { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); + statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; + } +} + +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol) +{ + FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + BIT_addBits(bitC, statePtr->value, nbBitsOut); + statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; +} + +MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) +{ + BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_flushBits(bitC); +} + + +/* FSE_getMaxNbBits() : + * Approximate maximum cost of a symbol, in bits. + * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16; +} + +/* FSE_bitCost() : + * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16; + U32 const threshold = (minNbBits+1) << 16; + assert(tableLog < 16); + assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */ + { U32 const tableSize = 1 << tableLog; + U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize); + U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */ + U32 const bitMultiplier = 1 << accuracyLog; + assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold); + assert(normalizedDeltaFromThreshold <= bitMultiplier); + return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold; + } +} + + +/* ====== Decompression ====== */ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSE_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif +#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE) +# error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE" +#endif + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG) +#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1) +#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2) +#define FSE_MIN_TABLELOG 5 + +#define FSE_TABLELOG_ABSOLUTE_MAX 15 +#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX +# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) + + +#endif /* FSE_STATIC_LINKING_ONLY */ + + diff --git a/src/bled/fse_bitstream.h b/src/bled/fse_bitstream.h new file mode 100644 index 00000000000..a6f3622923a --- /dev/null +++ b/src/bled/fse_bitstream.h @@ -0,0 +1,458 @@ +/* ****************************************************************** + * bitstream + * Part of FSE library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +/* +* This API consists of small unitary functions, which must be inlined for best performance. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + +/*-**************************************** +* Dependencies +******************************************/ +#include "zstd_mem.h" /* unaligned access routines */ +#include "zstd_compiler.h" /* UNLIKELY() */ +#include "zstd_error_private.h" /* error codes and messages */ + + +/*========================================= +* Target specific +=========================================*/ + +#define STREAM_ACCUMULATOR_MIN_32 25 +#define STREAM_ACCUMULATOR_MIN_64 57 +#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64)) + + +/*-****************************************** +* bitStream encoding API (write forward) +********************************************/ +/* bitStream can mix input from multiple sources. + * A critical property of these streams is that they encode and decode in **reverse** direction. + * So the first bit sequence you add will be the last to be read, like a LIFO stack. + */ +typedef struct { + size_t bitContainer; + unsigned bitPos; + char* startPtr; + char* ptr; + char* endPtr; +} BIT_CStream_t; + +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); + +/* Start with initCStream, providing the size of buffer to write into. +* bitStream will never write outside of this buffer. +* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. +* +* bits are first added to a local register. +* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Writing data into memory is an explicit operation, performed by the flushBits function. +* Hence keep track how many bits are potentially stored into local register to avoid register overflow. +* After a flushBits, a maximum of 7 bits might still be stored into local register. +* +* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. +* +* Last operation is to close the bitStream. +* The function returns the final size of CStream in bytes. +* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) +*/ + + +/*-******************************************** +* bitStream decoding API (read backward) +**********************************************/ +typedef struct { + size_t bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; + const char* limitPtr; +} BIT_DStream_t; + +typedef enum { BIT_DStream_unfinished = 0, + BIT_DStream_endOfBuffer = 1, + BIT_DStream_completed = 2, + BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ + /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + + +/* Start by invoking BIT_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). +*/ + + +/*-**************************************** +* unsafe API +******************************************/ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ + +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); +/* unsafe version; does not check buffer overflow */ + +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + + + +/*-************************************************************** +* Internal functions +****************************************************************/ +MEM_STATIC unsigned BIT_highbit32 (U32 val) +{ + assert(val != 0); + { +# if defined(_MSC_VER) /* Visual */ +# if STATIC_BMI2 == 1 + return _lzcnt_u32(val) ^ 31; +# else + unsigned long r = 0; + return _BitScanReverse(&r, val) ? (unsigned)r : 0; +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ + return __builtin_clz (val) ^ 31; +# elif defined(__ICCARM__) /* IAR Intrinsic */ + return 31 - __CLZ(val); +# else /* Software version */ + static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; +# endif + } +} + +/*===== Local Constants =====*/ +static const unsigned BIT_mask[] = { + 0, 1, 3, 7, 0xF, 0x1F, + 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, + 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, + 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, + 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF, + 0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */ +#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0])) + +/*-************************************************************** +* bitStream encoding +****************************************************************/ +/*! BIT_initCStream() : + * `dstCapacity` must be > sizeof(size_t) + * @return : 0 if success, + * otherwise an error code (can be tested using ERR_isError()) */ +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, + void* startPtr, size_t dstCapacity) +{ + bitC->bitContainer = 0; + bitC->bitPos = 0; + bitC->startPtr = (char*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer); + if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall); + return 0; +} + +/*! BIT_addBits() : + * can add up to 31 bits into `bitC`. + * Note : does not check for register overflow ! */ +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); + assert(nbBits < BIT_MASK_SIZE); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_addBitsFast() : + * works only if `value` is _clean_, + * meaning all high bits above nbBits are 0 */ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + assert((value>>nbBits) == 0); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= value << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_flushBitsFast() : + * assumption : bitContainer has not overflowed + * unsafe version; does not check buffer overflow */ +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_flushBits() : + * assumption : bitContainer has not overflowed + * safe version; check for buffer overflow, and prevents it. + * note : does not signal buffer overflow. + * overflow will be revealed later on using BIT_closeCStream() */ +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_closeCStream() : + * @return : size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) +{ + BIT_addBitsFast(bitC, 1, 1); /* endMark */ + BIT_flushBits(bitC); + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); +} + + +/*-******************************************************** +* bitStream decoding +**********************************************************/ +/*! BIT_initDStream() : + * Initialize a BIT_DStream_t. + * `bitD` : a pointer to an already allocated BIT_DStream_t structure. + * `srcSize` must be the *exact* size of the bitStream, in bytes. + * @return : size of stream (== srcSize), or an errorCode if a problem is detected + */ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + bitD->start = (const char*)srcBuffer; + bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer); + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + } else { + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + ZSTD_FALLTHROUGH; + + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + ZSTD_FALLTHROUGH; + + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + ZSTD_FALLTHROUGH; + + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + ZSTD_FALLTHROUGH; + + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + ZSTD_FALLTHROUGH; + + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; + ZSTD_FALLTHROUGH; + + default: break; + } + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ + } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + } + + return srcSize; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +{ + U32 const regMask = sizeof(bitContainer)*8 - 1; + /* if start > regMask, bitstream is corrupted, and result is undefined */ + assert(nbBits < BIT_MASK_SIZE); + /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better + * than accessing memory. When bmi2 instruction is not present, we consider + * such cpus old (pre-Haswell, 2013) and their performance is not of that + * importance. + */ +#if defined(__x86_64__) || defined(_M_X86) + return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1); +#else + return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; +#endif +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ +#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 + return _bzhi_u64(bitContainer, nbBits); +#else + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +#endif +} + +/*! BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted */ +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +{ + /* arbitrate between double-shift and shift+mask */ +#if 1 + /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8, + * bitstream is likely corrupted, and result is undefined */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#else + /* this code path is slower on my os-x laptop */ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask); +#endif +} + +/*! BIT_lookBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +{ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + assert(nbBits >= 1); + return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); +} + +MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BIT_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. */ +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_readBitsFast() : + * unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBitsFast(bitD, nbBits); + assert(nbBits >= 1); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_reloadDStreamFast() : + * Similar to BIT_reloadDStream(), but with two differences: + * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! + * 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this + * point you must use BIT_reloadDStream() to reload. + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) +{ + if (UNLIKELY(bitD->ptr < bitD->limitPtr)) + return BIT_DStream_overflow; + assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; +} + +/*! BIT_reloadDStream() : + * Refill `bitD` from buffer previously set in BIT_initDStream() . + * This function is safe, it guarantees it will not read beyond src buffer. + * @return : status of `BIT_DStream_t` internal register. + * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ + return BIT_DStream_overflow; + + if (bitD->ptr >= bitD->limitPtr) { + return BIT_reloadDStreamFast(bitD); + } + if (bitD->ptr == bitD->start) { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + /* start < ptr < limitPtr */ + { U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */ + return result; + } +} + +/*! BIT_endOfDStream() : + * @return : 1 if DStream has _exactly_ reached its end (all bits consumed). + */ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + + +#endif /* BITSTREAM_H_MODULE */ diff --git a/src/bled/fse_decompress.c b/src/bled/fse_decompress.c new file mode 100644 index 00000000000..610898e8683 --- /dev/null +++ b/src/bled/fse_decompress.c @@ -0,0 +1,390 @@ +/* ****************************************************************** + * FSE : Finite State Entropy decoder + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* ************************************************************** +* Includes +****************************************************************/ +//#include "debug.h" /* assert */ +#include "fse_bitstream.h" +#include "zstd_compiler.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "zstd_error_private.h" +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ +FSE_DTable* FSE_createDTable (unsigned tableLog) +{ + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); +} + +void FSE_freeDTable (FSE_DTable* dt) +{ + ZSTD_free(dt); +} + +static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); + U16* symbolNext = (U16*)workSpace; + BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1); + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge); + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s<maxSV1; s++) { + if (normalizedCounter[s]==-1) { + tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; + symbolNext[s] = 1; + } else { + if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s<maxSV1; ++s, sv += add) { + int i; + int const n = normalizedCounter[s]; + MEM_write64(spread + pos, sv); + for (i = 8; i < n; i += 8) { + MEM_write64(spread + pos + i, sv); + } + pos += n; + } + } + /* Now we spread those positions across the table. + * The benefit of doing it in two stages is that we avoid the the + * variable size inner loop, which caused lots of branch misses. + * Now we can run through all the positions without any branch misses. + * We unroll the loop twice, since that is what emperically worked best. + */ + { + size_t position = 0; + size_t s; + size_t const unroll = 2; + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableDecode[uPosition].symbol = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; + } + assert(position == 0); + } + } else { + U32 const tableMask = tableSize-1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s<maxSV1; s++) { + int i; + for (i=0; i<normalizedCounter[s]; i++) { + tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s; + position = (position + step) & tableMask; + while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; u<tableSize; u++) { + FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol); + U32 const nextState = symbolNext[symbol]++; + tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) ); + tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); + } } + + return 0; +} + +size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) +{ + return FSE_buildDTable_internal(dt, normalizedCounter, maxSymbolValue, tableLog, workSpace, wkspSize); +} + + +#ifndef FSE_COMMONDEFS_ONLY + +/*-******************************************************* +* Decompression (Byte symbols) +*********************************************************/ +size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const cell = (FSE_decode_t*)dPtr; + + DTableH->tableLog = 0; + DTableH->fastMode = 0; + + cell->newState = 0; + cell->symbol = symbolValue; + cell->nbBits = 0; + + return 0; +} + + +size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSV1 = tableMask+1; + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* Build Decoding Table */ + DTableH->tableLog = (U16)nbBits; + DTableH->fastMode = 1; + for (s=0; s<maxSV1; s++) { + dinfo[s].newState = 0; + dinfo[s].symbol = (BYTE)s; + dinfo[s].nbBits = (BYTE)nbBits; + } + + return 0; +} + +FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic( + void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt, const unsigned fast) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const omax = op + maxDstSize; + BYTE* const olimit = omax-3; + + BIT_DStream_t bitD; + FSE_DState_t state1; + FSE_DState_t state2; + + /* Init */ + CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize)); + + FSE_initDState(&state1, &bitD, dt); + FSE_initDState(&state2, &bitD, dt); + +#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) + + /* 4 symbols per loop */ + for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) & (op<olimit) ; op+=4) { + op[0] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state1); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state2); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } } + + return op-ostart; +} + + +size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); +} + + +size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0); +} + +typedef struct { + short ncount[FSE_MAX_SYMBOL_VALUE + 1]; + FSE_DTable dtable[1]; /* Dynamically sized */ +} FSE_DecompressWksp; + + +FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( + void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize, + unsigned maxLog, void* workSpace, size_t wkspSize, + int bmi2) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace; + + DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0); + if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC); + + /* normal FSE decoding mode */ + { + size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); + if (FSE_isError(NCountLength)) return NCountLength; + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + assert(NCountLength <= cSrcSize); + ip += NCountLength; + cSrcSize -= NCountLength; + } + + if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); + workSpace = wksp->dtable + FSE_DTABLE_SIZE_U32(tableLog); + wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + + CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); + + { + const void* ptr = wksp->dtable; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1); + return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0); + } +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +TARGET_ATTRIBUTE("bmi2") static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); +} +#endif + +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); + } +#endif + (void)bmi2; + return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); +} + + +typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; + + + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/src/bled/huf.h b/src/bled/huf.h new file mode 100644 index 00000000000..91e33c5417f --- /dev/null +++ b/src/bled/huf.h @@ -0,0 +1,344 @@ +/* ****************************************************************** + * huff0 huffman codec, + * part of Finite State Entropy library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +#ifndef HUF_H_298734234 +#define HUF_H_298734234 + +/* *** Dependencies *** */ +#include "zstd_deps.h" /* size_t */ + +# define HUF_PUBLIC_API + +/* ========================== */ +/* *** simple functions *** */ +/* ========================== */ + +/** HUF_compress() : + * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. + * 'dst' buffer must be already allocated. + * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). + * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. + * @return : size of compressed data (<= `dstCapacity`). + * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + * if HUF_isError(return), compression failed (more details using HUF_getErrorName()) + */ +HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/** HUF_decompress() : + * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', + * into already allocated buffer 'dst', of minimum size 'dstSize'. + * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. + * Note : in contrast with FSE, HUF_decompress can regenerate + * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, + * because it knows size to regenerate (originalSize). + * @return : size of regenerated data (== originalSize), + * or an error code, which can be tested using HUF_isError() + */ +HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize); + + +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +/* *** Advanced function *** */ + +/** HUF_compress2() : + * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`. + * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX . + * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ +HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog); + +/** HUF_compress4X_wksp() : + * Same as HUF_compress2(), but uses externally allocated `workSpace`. + * `workspace` must be at least as large as HUF_WORKSPACE_SIZE */ +#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */) +#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64)) +HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize); + +#endif /* HUF_H_298734234 */ + +/* ****************************************************************** + * WARNING !! + * The following section contains advanced and experimental definitions + * which shall never be used in the context of a dynamic library, + * because they are not guaranteed to remain stable in the future. + * Only consider them in association with static linking. + * *****************************************************************/ +#if !defined(HUF_H_HUF_STATIC_LINKING_ONLY) +#define HUF_H_HUF_STATIC_LINKING_ONLY + +/* *** Dependencies *** */ +#include "zstd_mem.h" /* U32 */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" + + +/* *** Constants *** */ +#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */ +#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ +#define HUF_SYMBOLVALUE_MAX 255 + +#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" +#endif + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */ +typedef size_t HUF_CElt; /* consider it an incomplete type */ +#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t)) +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */ + +/* static allocation of HUF's DTable */ +typedef U32 HUF_DTable; +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) +#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } +#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) } + + +/* **************************************** +* Advanced decompression functions +******************************************/ +HUF_PUBLIC_API size_t HUF_decompress4X1(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 +HUF_PUBLIC_API size_t HUF_decompress4X2(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ +#endif + +HUF_PUBLIC_API size_t HUF_decompress4X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ +HUF_PUBLIC_API size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ +HUF_PUBLIC_API size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */ +HUF_PUBLIC_API size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +HUF_PUBLIC_API size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 +HUF_PUBLIC_API size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ +HUF_PUBLIC_API size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ +#endif + + +/* **************************************** + * HUF detailed API + * ****************************************/ + +/*! HUF_compress() does the following: + * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") + * 2. (optional) refine tableLog using HUF_optimalTableLog() + * 3. build Huffman table from count using HUF_buildCTable() + * 4. save Huffman table to memory buffer using HUF_writeCTable() + * 5. encode the data stream using HUF_compress4X_usingCTable() + * + * The following API allows targeting specific sub-functions for advanced tasks. + * For example, it's possible to compress several blocks using the same 'CTable', + * or to save and regenerate 'CTable' using external methods. + */ +HUF_PUBLIC_API unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); +HUF_PUBLIC_API size_t HUF_buildCTable(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */ +HUF_PUBLIC_API size_t HUF_writeCTable(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); +HUF_PUBLIC_API size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); +HUF_PUBLIC_API size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +HUF_PUBLIC_API size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2); +HUF_PUBLIC_API size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); +HUF_PUBLIC_API int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); + +typedef enum { + HUF_repeat_none, /**< Cannot use the previous table */ + HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ + HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ + } HUF_repeat; +/** HUF_compress4X_repeat() : + * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +HUF_PUBLIC_API size_t HUF_compress4X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible); + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. + */ +#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1) +#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +HUF_PUBLIC_API size_t HUF_buildCTable_wksp(HUF_CElt* tree, + const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, + void* workSpace, size_t wkspSize); + +/*! HUF_readStats() : + * Read compact Huffman tree, saved by HUF_writeCTable(). + * `huffWeight` is destination buffer. + * @return : size read from `src` , or an error Code . + * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ +HUF_PUBLIC_API size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); + +/*! HUF_readStats_wksp() : + * Same as HUF_readStats() but takes an external workspace which must be + * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1) +#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +HUF_PUBLIC_API size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workspace, size_t wkspSize, + int bmi2); + +/** HUF_readCTable() : + * Loading a CTable saved with HUF_writeCTable() */ +HUF_PUBLIC_API size_t HUF_readCTable(HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); + +/** HUF_getNbBitsFromCTable() : + * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX + * Note 1 : is not inlined, as HUF_CElt definition is private */ +HUF_PUBLIC_API U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue); + +/* + * HUF_decompress() does the following: + * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics + * 2. build Huffman table from save, using HUF_readDTableX?() + * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable() + */ + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +HUF_PUBLIC_API U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize); + +/** + * The minimum workspace size for the `workSpace` used in + * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp(). + * + * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when + * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15. + * Buffer overflow errors may potentially occur if code modifications result in + * a required workspace size greater than that specified in the following + * macro. + */ +#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9)) +#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) + +#ifndef HUF_FORCE_DECOMPRESS_X2 +HUF_PUBLIC_API size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize); +HUF_PUBLIC_API size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +HUF_PUBLIC_API size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize); +HUF_PUBLIC_API size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); +#endif + +HUF_PUBLIC_API size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#ifndef HUF_FORCE_DECOMPRESS_X2 +HUF_PUBLIC_API size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +HUF_PUBLIC_API size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif + + +/* ====================== */ +/* single stream variants */ +/* ====================== */ + +HUF_PUBLIC_API size_t HUF_compress1X(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); +HUF_PUBLIC_API size_t HUF_compress1X_wksp(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U64 U64 */ +HUF_PUBLIC_API size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +HUF_PUBLIC_API size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2); +/** HUF_compress1X_repeat() : + * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +HUF_PUBLIC_API size_t HUF_compress1X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible); + +HUF_PUBLIC_API size_t HUF_decompress1X1(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 +HUF_PUBLIC_API size_t HUF_decompress1X2(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ +#endif + +HUF_PUBLIC_API size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); +HUF_PUBLIC_API size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); +#ifndef HUF_FORCE_DECOMPRESS_X2 +HUF_PUBLIC_API size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +HUF_PUBLIC_API size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +HUF_PUBLIC_API size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ +HUF_PUBLIC_API size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ +#endif + +HUF_PUBLIC_API size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ +#ifndef HUF_FORCE_DECOMPRESS_X2 +HUF_PUBLIC_API size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +HUF_PUBLIC_API size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif + +/* BMI2 variants. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +#ifndef HUF_FORCE_DECOMPRESS_X2 +HUF_PUBLIC_API size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +#endif +HUF_PUBLIC_API size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +#ifndef HUF_FORCE_DECOMPRESS_X2 +HUF_PUBLIC_API size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +HUF_PUBLIC_API size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2); +#endif + +#endif /* HUF_STATIC_LINKING_ONLY */ + diff --git a/src/bled/huf_decompress.c b/src/bled/huf_decompress.c new file mode 100644 index 00000000000..5a019c7d112 --- /dev/null +++ b/src/bled/huf_decompress.c @@ -0,0 +1,1747 @@ +/* ****************************************************************** + * huff0 huffman decoder, + * part of Finite State Entropy library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Dependencies +****************************************************************/ +#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "zstd_compiler.h" +#include "fse_bitstream.h" /* BIT_* */ +#include "fse.h" /* to compress headers */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_error_private.h" +#include "zstd_internal.h" + +/* ************************************************************** +* Constants +****************************************************************/ + +#define HUF_DECODER_FAST_TABLELOG 11 + +/* ************************************************************** +* Macros +****************************************************************/ + +/* These two optional macros force the use one way or another of the two + * Huffman decompression implementations. You can't force in both directions + * at the same time. + */ +#if defined(HUF_FORCE_DECOMPRESS_X1) && \ + defined(HUF_FORCE_DECOMPRESS_X2) +#error "Cannot force the use of the X1 and X2 decoders at the same time!" +#endif + +# define HUF_ASM_SUPPORTED 0 + +/* HUF_DISABLE_ASM: Disables all ASM implementations. */ +#define HUF_ENABLE_ASM_X86_64_BMI2 0 + +#if HUF_ENABLE_ASM_X86_64_BMI2 && DYNAMIC_BMI2 +# define HUF_ASM_X86_64_BMI2_ATTRS TARGET_ATTRIBUTE("bmi2") +#else +# define HUF_ASM_X86_64_BMI2_ATTRS +#endif + +#define HUF_EXTERN_C +#define HUF_ASM_DECL HUF_EXTERN_C + +#if DYNAMIC_BMI2 || (HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)) +# define HUF_NEED_BMI2_FUNCTION 1 +#else +# define HUF_NEED_BMI2_FUNCTION 0 +#endif + +#if !(HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)) +# define HUF_NEED_DEFAULT_FUNCTION 1 +#else +# define HUF_NEED_DEFAULT_FUNCTION 0 +#endif + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError + + +/* ************************************************************** +* Byte alignment for workSpace management +****************************************************************/ +#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1) +#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) + + +/* ************************************************************** +* BMI2 Variant Wrappers +****************************************************************/ +#if DYNAMIC_BMI2 + +#define HUF_DGEN(fn) \ + \ + static size_t fn##_default( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static TARGET_ATTRIBUTE("bmi2") size_t fn##_bmi2( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + { \ + if (bmi2) { \ + return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#else + +#define HUF_DGEN(fn) \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + { \ + (void)bmi2; \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#endif + + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + ZSTD_memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + +#if HUF_ENABLE_ASM_X86_64_BMI2 + +static size_t HUF_initDStream(BYTE const* ip) { + BYTE const lastByte = ip[7]; + size_t const bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + size_t const value = MEM_readLEST(ip) | 1; + assert(bitsConsumed <= 8); + return value << bitsConsumed; +} +typedef struct { + BYTE const* ip[4]; + BYTE* op[4]; + U64 bits[4]; + void const* dt; + BYTE const* ilimit; + BYTE* oend; + BYTE const* iend[4]; +} HUF_DecompressAsmArgs; + +/** + * Initializes args for the asm decoding loop. + * @returns 0 on success + * 1 if the fallback implementation should be used. + * Or an error code on failure. + */ +static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) +{ + void const* dt = DTable + 1; + U32 const dtLog = HUF_getDTableDesc(DTable).tableLog; + + const BYTE* const ilimit = (const BYTE*)src + 6 + 8; + + BYTE* const oend = (BYTE*)dst + dstSize; + + /* We're assuming x86-64 BMI2 - assure that this is the case. */ + assert(MEM_isLittleEndian() && !MEM_32bits()); + + /* strict minimum : jump table + 1 byte per stream */ + if (srcSize < 10) + return ERROR(corruption_detected); + + /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers. + * If table log is not correct at this point, fallback to the old decoder. + * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder. + */ + if (dtLog != HUF_DECODER_FAST_TABLELOG) + return 1; + + /* Read the jump table. */ + { + const BYTE* const istart = (const BYTE*)src; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = srcSize - (length1 + length2 + length3 + 6); + args->iend[0] = istart + 6; /* jumpTable */ + args->iend[1] = args->iend[0] + length1; + args->iend[2] = args->iend[1] + length2; + args->iend[3] = args->iend[2] + length3; + + /* HUF_initDStream() requires this, and this small of an input + * won't benefit from the ASM loop anyways. + * length1 must be >= 16 so that ip[0] >= ilimit before the loop + * starts. + */ + if (length1 < 16 || length2 < 8 || length3 < 8 || length4 < 8) + return 1; + if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */ + } + /* ip[] contains the position that is currently loaded into bits[]. */ + args->ip[0] = args->iend[1] - sizeof(U64); + args->ip[1] = args->iend[2] - sizeof(U64); + args->ip[2] = args->iend[3] - sizeof(U64); + args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64); + + /* op[] contains the output pointers. */ + args->op[0] = (BYTE*)dst; + args->op[1] = args->op[0] + (dstSize+3)/4; + args->op[2] = args->op[1] + (dstSize+3)/4; + args->op[3] = args->op[2] + (dstSize+3)/4; + + /* No point to call the ASM loop for tiny outputs. */ + if (args->op[3] >= oend) + return 1; + + /* bits[] is the bit container. + * It is read from the MSB down to the LSB. + * It is shifted left as it is read, and zeros are + * shifted in. After the lowest valid bit a 1 is + * set, so that CountTrailingZeros(bits[]) can be used + * to count how many bits we've consumed. + */ + args->bits[0] = HUF_initDStream(args->ip[0]); + args->bits[1] = HUF_initDStream(args->ip[1]); + args->bits[2] = HUF_initDStream(args->ip[2]); + args->bits[3] = HUF_initDStream(args->ip[3]); + + /* If ip[] >= ilimit, it is guaranteed to be safe to + * reload bits[]. It may be beyond its section, but is + * guaranteed to be valid (>= istart). + */ + args->ilimit = ilimit; + + args->oend = oend; + args->dt = dt; + + return 0; +} + +static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs const* args, int stream, BYTE* segmentEnd) +{ + /* Validate that we haven't overwritten. */ + if (args->op[stream] > segmentEnd) + return ERROR(corruption_detected); + /* Validate that we haven't read beyond iend[]. + * Note that ip[] may be < iend[] because the MSB is + * the next bit to read, and we may have consumed 100% + * of the stream, so down to iend[i] - 8 is valid. + */ + if (args->ip[stream] < args->iend[stream] - 8) + return ERROR(corruption_detected); + + /* Construct the BIT_DStream_t. */ + bit->bitContainer = MEM_readLE64(args->ip[stream]); + bit->bitsConsumed = ZSTD_countTrailingZeros((size_t)args->bits[stream]); + bit->start = (const char*)args->iend[0]; + bit->limitPtr = bit->start + sizeof(size_t); + bit->ptr = (const char*)args->ip[stream]; + + return 0; +} +#endif + + +#ifndef HUF_FORCE_DECOMPRESS_X2 + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ +typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */ + +/** + * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at + * a time. + */ +static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { + U64 D4; + if (MEM_isLittleEndian()) { + D4 = (symbol << 8) + nbBits; + } else { + D4 = symbol + (nbBits << 8); + } + D4 *= 0x0001000100010001ULL; + return D4; +} + +/** + * Increase the tableLog to targetTableLog and rescales the stats. + * If tableLog > targetTableLog this is a no-op. + * @returns New tableLog + */ +static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog) +{ + if (tableLog > targetTableLog) + return tableLog; + if (tableLog < targetTableLog) { + U32 const scale = targetTableLog - tableLog; + U32 s; + /* Increase the weight for all non-zero probability symbols by scale. */ + for (s = 0; s < nbSymbols; ++s) { + huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale); + } + /* Update rankVal to reflect the new weights. + * All weights except 0 get moved to weight + scale. + * Weights [1, scale] are empty. + */ + for (s = targetTableLog; s > scale; --s) { + rankVal[s] = rankVal[s - scale]; + } + for (s = scale; s > 0; --s) { + rankVal[s] = 0; + } + } + return targetTableLog; +} + +typedef struct { + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; +} HUF_ReadDTableX1_Workspace; + + +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) +{ + return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0); +} + +size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2) +{ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; + HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; + + DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); + if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); + + DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2); + if (HUF_isError(iSize)) return iSize; + + + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog + 1; + U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG); + tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Compute symbols and rankStart given rankVal: + * + * rankVal already contains the number of values of each weight. + * + * symbols contains the symbols ordered by weight. First are the rankVal[0] + * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. + * symbols[0] is filled (but unused) to avoid a branch. + * + * rankStart contains the offset where each rank belongs in the DTable. + * rankStart[0] is not filled because there are no entries in the table for + * weight 0. + */ + { + int n; + int nextRankStart = 0; + int const unroll = 4; + int const nLimit = (int)nbSymbols - unroll + 1; + for (n=0; n<(int)tableLog+1; n++) { + U32 const curr = nextRankStart; + nextRankStart += wksp->rankVal[n]; + wksp->rankStart[n] = curr; + } + for (n=0; n < nLimit; n += unroll) { + int u; + for (u=0; u < unroll; ++u) { + size_t const w = wksp->huffWeight[n+u]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); + } + } + for (; n < (int)nbSymbols; ++n) { + size_t const w = wksp->huffWeight[n]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; + } + } + + /* fill DTable + * We fill all entries of each weight in order. + * That way length is a constant for each iteration of the outter loop. + * We can switch based on the length to a different inner loop which is + * optimized for that particular case. + */ + { + U32 w; + int symbol=wksp->rankVal[0]; + int rankStart=0; + for (w=1; w<tableLog+1; ++w) { + int const symbolCount = wksp->rankVal[w]; + int const length = (1 << w) >> 1; + int uStart = rankStart; + BYTE const nbBits = (BYTE)(tableLog + 1 - w); + int s; + int u; + switch (length) { + case 1: + for (s=0; s<symbolCount; ++s) { + HUF_DEltX1 D; + D.byte = wksp->symbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart] = D; + uStart += 1; + } + break; + case 2: + for (s=0; s<symbolCount; ++s) { + HUF_DEltX1 D; + D.byte = wksp->symbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart+0] = D; + dt[uStart+1] = D; + uStart += 2; + } + break; + case 4: + for (s=0; s<symbolCount; ++s) { + U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + uStart += 4; + } + break; + case 8: + for (s=0; s<symbolCount; ++s) { + U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + MEM_write64(dt + uStart + 4, D4); + uStart += 8; + } + break; + default: + for (s=0; s<symbolCount; ++s) { + U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits); + for (u=0; u < length; u += 16) { + MEM_write64(dt + uStart + u + 0, D4); + MEM_write64(dt + uStart + u + 4, D4); + MEM_write64(dt + uStart + u + 8, D4); + MEM_write64(dt + uStart + u + 12, D4); + } + assert(u == length); + uStart += length; + } + break; + } + symbol += symbolCount; + rankStart += symbolCount * length; + } + } + return iSize; +} + +FORCE_INLINE_TEMPLATE BYTE +HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ + *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) + +#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) + +HINT_INLINE size_t +HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + if ((pEnd - p) > 3) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_1(p, bitDPtr); + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + } + } + + /* [0-3] symbols remaining */ + if (MEM_32bits()) + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + return pEnd-pStart; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstSize; + const void* dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); + + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - 3; + const void* const dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + U32 endSignal = 1; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit) ; ) { + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_1(op1, &bitD1); + HUF_DECODE_SYMBOLX1_1(op2, &bitD2); + HUF_DECODE_SYMBOLX1_1(op3, &bitD3); + HUF_DECODE_SYMBOLX1_1(op4, &bitD4); + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_0(op1, &bitD1); + HUF_DECODE_SYMBOLX1_0(op2, &bitD2); + HUF_DECODE_SYMBOLX1_0(op3, &bitD3); + HUF_DECODE_SYMBOLX1_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + } + } + + /* check corruption */ + /* note : should not be necessary : op# advance in lock step, and we control op4. + * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static TARGET_ATTRIBUTE("bmi2") +size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +#if HUF_NEED_DEFAULT_FUNCTION +static +size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +#if HUF_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args); + +static HUF_ASM_X86_64_BMI2_ATTRS +size_t +HUF_decompress4X1_usingDTable_internal_bmi2_asm( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + void const* dt = DTable + 1; + const BYTE* const iend = (const BYTE*)cSrc + 6; + BYTE* const oend = (BYTE*)dst + dstSize; + HUF_DecompressAsmArgs args; + { + size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init asm args"); + if (ret != 0) + return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); + } + + assert(args.ip[0] >= args.ilimit); + HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(&args); + + /* Our loop guarantees that ip[] >= ilimit and that we haven't + * overwritten any op[]. + */ + assert(args.ip[0] >= iend); + assert(args.ip[1] >= iend); + assert(args.ip[2] >= iend); + assert(args.ip[3] >= iend); + assert(args.op[3] <= oend); + (void)iend; + + /* finish bit streams one by one. */ + { + size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + /* Decompress and validate that we've produced exactly the expected length. */ + args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) return ERROR(corruption_detected); + } + } + + /* decoded size */ + return dstSize; +} +#endif /* HUF_ENABLE_ASM_X86_64_BMI2 */ + +typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize, + const void *cSrc, + size_t cSrcSize, + const HUF_DTable *DTable); + +HUF_DGEN(HUF_decompress1X1_usingDTable_internal) + +static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { +# if HUF_ENABLE_ASM_X86_64_BMI2 + return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); +# else + return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); +# endif + } +#else + (void)bmi2; +#endif + +#if HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); +#else + return HUF_decompress4X1_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable); +#endif +} + + +size_t HUF_decompress1X1_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); +} + + +size_t HUF_decompress4X1_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int bmi2) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); +} + +size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0); +} + + +#endif /* HUF_FORCE_DECOMPRESS_X2 */ + + +#ifndef HUF_FORCE_DECOMPRESS_X1 + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ + +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ +typedef struct { BYTE symbol; } sortedSymbol_t; +typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; +typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; + +/** + * Constructs a HUF_DEltX2 in a U32. + */ +static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + U32 seq; + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3); + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32)); + if (MEM_isLittleEndian()) { + seq = level == 1 ? symbol : (baseSeq + (symbol << 8)); + return seq + (nbBits << 16) + ((U32)level << 24); + } else { + seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol); + return (seq << 16) + (nbBits << 8) + (U32)level; + } +} + +/** + * Constructs a HUF_DEltX2. + */ +static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + HUF_DEltX2 DElt; + U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val)); + ZSTD_memcpy(&DElt, &val, sizeof(val)); + return DElt; +} + +/** + * Constructs 2 HUF_DEltX2s and packs them into a U64. + */ +static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level) +{ + U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + return (U64)DElt + ((U64)DElt << 32); +} + +/** + * Fills the DTable rank with all the symbols from [begin, end) that are each + * nbBits long. + * + * @param DTableRank The start of the rank in the DTable. + * @param begin The first symbol to fill (inclusive). + * @param end The last symbol to fill (exclusive). + * @param nbBits Each symbol is nbBits long. + * @param tableLog The table log. + * @param baseSeq If level == 1 { 0 } else { the first level symbol } + * @param level The level in the table. Must be 1 or 2. + */ +static void HUF_fillDTableX2ForWeight( + HUF_DEltX2* DTableRank, + sortedSymbol_t const* begin, sortedSymbol_t const* end, + U32 nbBits, U32 tableLog, + U16 baseSeq, int const level) +{ + U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */); + const sortedSymbol_t* ptr; + assert(level >= 1 && level <= 2); + switch (length) { + case 1: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + *DTableRank++ = DElt; + } + break; + case 2: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + DTableRank[0] = DElt; + DTableRank[1] = DElt; + DTableRank += 2; + } + break; + case 4: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + DTableRank += 4; + } + break; + case 8: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + DTableRank += 8; + } + break; + default: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + HUF_DEltX2* const DTableRankEnd = DTableRank + length; + for (; DTableRank != DTableRankEnd; DTableRank += 8) { + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + } + } + break; + } +} + +/* HUF_fillDTableX2Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits, + const U32* rankVal, const int minWeight, const int maxWeight1, + const sortedSymbol_t* sortedSymbols, U32 const* rankStart, + U32 nbBitsBaseline, U16 baseSeq) +{ + /* Fill skipped values (all positions up to rankVal[minWeight]). + * These are positions only get a single symbol because the combined weight + * is too large. + */ + if (minWeight>1) { + U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */); + U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1); + int const skipSize = rankVal[minWeight]; + assert(length > 1); + assert((U32)skipSize < length); + switch (length) { + case 2: + assert(skipSize == 1); + ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2)); + break; + case 4: + assert(skipSize <= 4); + ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2)); + break; + default: + { + int i; + for (i = 0; i < skipSize; i += 8) { + ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2)); + } + } + } + } + + /* Fill each of the second level symbols by weight. */ + { + int w; + for (w = minWeight; w < maxWeight1; ++w) { + int const begin = rankStart[w]; + int const end = rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + U32 const totalBits = nbBits + consumedBits; + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedSymbols + begin, sortedSymbols + end, + totalBits, targetLog, + baseSeq, /* level */ 2); + } + } +} + +static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, + const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32* const rankVal = rankValOrigin[0]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + int w; + int const wEnd = (int)maxWeight + 1; + + /* Fill DTable in order of weight. */ + for (w = 1; w < wEnd; ++w) { + int const begin = (int)rankStart[w]; + int const end = (int)rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + + if (targetLog-nbBits >= minBits) { + /* Enough room for a second symbol. */ + int start = rankVal[w]; + U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */); + int minWeight = nbBits + scaleLog; + int s; + if (minWeight < 1) minWeight = 1; + /* Fill the DTable for every symbol of weight w. + * These symbols get at least 1 second symbol. + */ + for (s = begin; s != end; ++s) { + HUF_fillDTableX2Level2( + DTable + start, targetLog, nbBits, + rankValOrigin[nbBits], minWeight, wEnd, + sortedList, rankStart, + nbBitsBaseline, sortedList[s].symbol); + start += length; + } + } else { + /* Only a single symbol. */ + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedList + begin, sortedList + end, + nbBits, targetLog, + /* baseSeq */ 0, /* level */ 1); + } + } +} + +typedef struct { + rankValCol_t rankVal[HUF_TABLELOG_MAX]; + U32 rankStats[HUF_TABLELOG_MAX + 1]; + U32 rankStart0[HUF_TABLELOG_MAX + 3]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; +} HUF_ReadDTableX2_Workspace; + +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readDTableX2_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0); +} + +size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, int bmi2) +{ + U32 tableLog, maxW, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + U32 *rankStart; + + HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace; + + if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC); + + rankStart = wksp->rankStart0 + 1; + ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats)); + ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0)); + + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ + if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), bmi2); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG; + + /* find maxWeight */ + for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; w<maxW+1; w++) { + U32 curr = nextRankStart; + nextRankStart += wksp->rankStats[w]; + rankStart[w] = curr; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + rankStart[maxW+1] = nextRankStart; + } + + /* sort symbols by weight */ + { U32 s; + for (s=0; s<nbSymbols; s++) { + U32 const w = wksp->weightList[s]; + U32 const r = rankStart[w]++; + wksp->sortedSymbol[r].symbol = (BYTE)s; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } + + /* Build rankVal */ + { U32* const rankVal0 = wksp->rankVal[0]; + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; w<maxW+1; w++) { + U32 curr = nextRankVal; + nextRankVal += wksp->rankStats[w] << (w+rescale); + rankVal0[w] = curr; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = wksp->rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } + +#if defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstringop-overflow" +#endif + HUF_fillDTableX2(dt, maxTableLog, + wksp->sortedSymbol, + wksp->rankStart0, wksp->rankVal, maxW, + tableLog+1); +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 1); + if (dt[val].length==1) { + BIT_skipBits(DStream, dt[val].nbBits); + } else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); + } + } + return 1; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +HINT_INLINE size_t +HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, + const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) { + if (dtLog <= 11 && MEM_64bits()) { + /* up to 10 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) { + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } else { + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } + } + + /* closer to end : up to 2 symbols at a time */ + if ((size_t)(pEnd - p) >= 2) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + } + + if (p < pEnd) + p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - (sizeof(size_t)-1); + const void* const dtPtr = DTable+1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal = 1; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit); ) { +#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; +#else + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = (U32)LIKELY( + (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); +#endif + } + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static TARGET_ATTRIBUTE("bmi2") +size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +#if HUF_NEED_DEFAULT_FUNCTION +static +size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +#if HUF_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args); + +static HUF_ASM_X86_64_BMI2_ATTRS size_t +HUF_decompress4X2_usingDTable_internal_bmi2_asm( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) { + void const* dt = DTable + 1; + const BYTE* const iend = (const BYTE*)cSrc + 6; + BYTE* const oend = (BYTE*)dst + dstSize; + HUF_DecompressAsmArgs args; + { + size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init asm args"); + if (ret != 0) + return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); + } + + assert(args.ip[0] >= args.ilimit); + HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(&args); + + /* note : op4 already verified within main loop */ + assert(args.ip[0] >= iend); + assert(args.ip[1] >= iend); + assert(args.ip[2] >= iend); + assert(args.ip[3] >= iend); + assert(args.op[3] <= oend); + (void)iend; + + /* finish bitStreams one by one */ + { + size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) + return ERROR(corruption_detected); + } + } + + /* decoded size */ + return dstSize; +} +#endif /* HUF_ENABLE_ASM_X86_64_BMI2 */ + +static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { +# if HUF_ENABLE_ASM_X86_64_BMI2 + return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); +# else + return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); +# endif + } +#else + (void)bmi2; +#endif + +#if HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); +#else + return HUF_decompress4X2_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable); +#endif +} + +HUF_DGEN(HUF_decompress1X2_usingDTable_internal) + +size_t HUF_decompress1X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, + workSpace, wkspSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); +} + + +size_t HUF_decompress4X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int bmi2) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, + workSpace, wkspSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); +} + +size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0); +} + + +#endif /* HUF_FORCE_DECOMPRESS_X1 */ + + +/* ***********************************/ +/* Universal decompression selectors */ +/* ***********************************/ + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#endif +} + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#endif +} + + +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}}, /* Q==0 : impossible */ + {{0,0}, {1,1}}, /* Q==1 : impossible */ + {{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */ + {{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */ + {{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */ + {{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */ + {{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */ + {{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */ + {{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */ + {{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */ + {{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */ + {{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */ + {{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */ + {{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */ + {{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */ + {{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */ +}; +#endif + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + assert(dstSize > 0); + assert(dstSize <= 128*1024); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dstSize; + (void)cSrcSize; + return 0; +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dstSize; + (void)cSrcSize; + return 1; +#else + /* decoder timing evaluation */ + { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */ + return DTime1 < DTime0; + } +#endif +} + + +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, + size_t dstSize, const void* cSrc, + size_t cSrcSize, void* workSpace, + size_t wkspSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize): + HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#endif + } +} + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize); +#else + return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize): + HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize); +#endif + } +} + + +size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#endif +} + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); +} +#endif + +size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#endif +} + +size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) : + HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#endif + } +} + diff --git a/src/bled/libbb.h b/src/bled/libbb.h index 000a1a66d9b..7d17823663e 100644 --- a/src/bled/libbb.h +++ b/src/bled/libbb.h @@ -23,7 +23,6 @@ #include "platform.h" #include "msapi_utf8.h" -#include <assert.h> #include <ctype.h> #include <errno.h> #include <fcntl.h> diff --git a/src/bled/open_transformer.c b/src/bled/open_transformer.c index 4e420c7ec72..f85855eb3f6 100644 --- a/src/bled/open_transformer.c +++ b/src/bled/open_transformer.c @@ -206,12 +206,24 @@ static transformer_state_t *setup_transformer_on_fd(int fd, int fail_if_not_comp goto found_magic; } } + if (ENABLE_FEATURE_SEAMLESS_ZSTD + && xstate->magic.b16[0] == ZSTD_MAGIC1 + ) { + offset = -4; + xread(fd, &xstate->magic.b16[1], 2); + if (xstate->magic.b16[1] == ZSTD_MAGIC2) { + xstate->xformer = unpack_zstd_stream; + USE_FOR_NOMMU(xstate->xformer_prog = "unzstd";) + goto found_magic; + } + } /* No known magic seen */ if (fail_if_not_compressed) bb_error_msg_and_die("no gzip" IF_FEATURE_SEAMLESS_BZ2("/bzip2") IF_FEATURE_SEAMLESS_XZ("/xz") + IF_FEATURE_SEAMLESS_ZSTD("/zstd") " magic"); /* Some callers expect this function to "consume" fd diff --git a/src/bled/xxhash.c b/src/bled/xxhash.c new file mode 100644 index 00000000000..d8e84c68605 --- /dev/null +++ b/src/bled/xxhash.c @@ -0,0 +1,821 @@ +/* + * xxHash - Fast Hash algorithm + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - xxHash homepage: http://www.xxhash.com + * - xxHash source repository : https://github.com/Cyan4973/xxHash + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +*/ + + +/* ************************************* +* Tuning parameters +***************************************/ +/*!XXH_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. + * It can generate buggy code on targets which do not support unaligned memory accesses. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://stackoverflow.com/a/32095106/646947 for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) || \ + defined(__ICCARM__) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/*!XXH_ACCEPT_NULL_INPUT_POINTER : + * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. + * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. + * By default, this option is disabled. To enable it, uncomment below define : + */ +/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ + +/*!XXH_FORCE_NATIVE_FORMAT : + * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. + * Results are therefore identical for little-endian and big-endian CPU. + * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. + * Should endian-independence be of no importance for your application, you may set the #define below to 1, + * to improve speed for Big-endian CPU. + * This option has no impact on Little_Endian CPU. + */ +#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ +# define XXH_FORCE_NATIVE_FORMAT 0 +#endif + +/*!XXH_FORCE_ALIGN_CHECK : + * This is a minor performance trick, only useful with lots of very small keys. + * It means : check for aligned/unaligned input. + * The check costs one initial branch per hash; set to 0 when the input data + * is guaranteed to be aligned. + */ +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/* Modify the local functions below should you wish to use some other memory routines */ +/* for ZSTD_malloc(), ZSTD_free() */ +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* size_t, ZSTD_malloc, ZSTD_free, ZSTD_memcpy */ +static void* XXH_malloc(size_t s) { return ZSTD_malloc(s); } +static void XXH_free (void* p) { ZSTD_free(p); } +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return ZSTD_memcpy(dest,src,size); } + +#include "xxhash.h" + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#include "zstd_compiler.h" + + +/* ************************************* +* Basic Types +***************************************/ +#include "zstd_mem.h" /* BYTE, U32, U64, size_t */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } +static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign; + +static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } + +#else + +/* portable and safe solution. Generally efficient. + * see : http://stackoverflow.com/a/32095106/646947 + */ + +static U32 XXH_read32(const void* memPtr) +{ + U32 val; + ZSTD_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +static U64 XXH_read64(const void* memPtr) +{ + U64 val; + ZSTD_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +#if defined(__ICCARM__) +# include <intrinsics.h> +# define XXH_rotl32(x,r) __ROR(x,(32 - r)) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +#endif +# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +#endif + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +# define XXH_swap64 _byteswap_uint64 +#elif GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +# define XXH_swap64 __builtin_bswap64 +#else +static U32 XXH_swap32 (U32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +static U64 XXH_swap64 (U64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* ************************************* +* Architecture Macros +***************************************/ +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; + +/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ +#ifndef XXH_CPU_LITTLE_ENDIAN + static const int g_one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) +#endif + + +/* *************************** +* Memory reads +*****************************/ +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE_TEMPLATE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); + else + return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); +} + +FORCE_INLINE_TEMPLATE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +static U32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} + +FORCE_INLINE_TEMPLATE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); + else + return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); +} + +FORCE_INLINE_TEMPLATE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + +static U64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} + + +/* ************************************* +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************* +* Constants +***************************************/ +static const U32 PRIME32_1 = 2654435761U; +static const U32 PRIME32_2 = 2246822519U; +static const U32 PRIME32_3 = 3266489917U; +static const U32 PRIME32_4 = 668265263U; +static const U32 PRIME32_5 = 374761393U; + +static const U64 PRIME64_1 = 11400714785074694791ULL; +static const U64 PRIME64_2 = 14029467366897019727ULL; +static const U64 PRIME64_3 = 1609587929392839161ULL; +static const U64 PRIME64_4 = 9650029242287828579ULL; +static const U64 PRIME64_5 = 2870177450012600261ULL; + +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ************************** +* Utils +****************************/ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState) +{ + ZSTD_memcpy(dstState, srcState, sizeof(*dstState)); +} + +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState) +{ + ZSTD_memcpy(dstState, srcState, sizeof(*dstState)); +} + + +/* *************************** +* Simple Hash Functions +*****************************/ + +static U32 XXH32_round(U32 seed, U32 input) +{ + seed += input * PRIME32_2; + seed = XXH_rotl32(seed, 13); + seed *= PRIME32_1; + return seed; +} + +FORCE_INLINE_TEMPLATE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)16; + } +#endif + + if (len>=16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; + v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; + v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; + v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; + } while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p+4<=bEnd) { + h32 += XXH_get32bits(p) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p<bEnd) { + h32 += (*p) * PRIME32_5; + h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; + p++; + } + + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_CREATESTATE_STATIC(state); + XXH32_reset(state, seed); + XXH32_update(state, input, len); + return XXH32_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + +static U64 XXH64_round(U64 acc, U64 input) +{ + acc += input * PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= PRIME64_1; + return acc; +} + +static U64 XXH64_mergeRound(U64 acc, U64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * PRIME64_1 + PRIME64_4; + return acc; +} + +FORCE_INLINE_TEMPLATE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + U64 h64; +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; + v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; + v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; + v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; + } while (p<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + + } else { + h64 = seed + PRIME64_5; + } + + h64 += (U64) len; + + while (p+8<=bEnd) { + U64 const k1 = XXH64_round(0, XXH_get64bits(p)); + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) { + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p<bEnd) { + h64 ^= (*p) * PRIME64_5; + h64 = XXH_rotl64(h64, 11) * PRIME64_1; + p++; + } + + h64 ^= h64 >> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_CREATESTATE_STATIC(state); + XXH64_reset(state, seed); + XXH64_update(state, input, len); + return XXH64_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + +/* ************************************************** +* Advanced Hash Functions +****************************************************/ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + + +/*** Hash feed ***/ + +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) +{ + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + ZSTD_memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ + state.v1 = seed + PRIME32_1 + PRIME32_2; + state.v2 = seed + PRIME32_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME32_1; + ZSTD_memcpy(statePtr, &state, sizeof(state)); + return XXH_OK; +} + + +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) +{ + XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + ZSTD_memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ + state.v1 = seed + PRIME64_1 + PRIME64_2; + state.v2 = seed + PRIME64_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME64_1; + ZSTD_memcpy(statePtr, &state, sizeof(state)); + return XXH_OK; +} + + +FORCE_INLINE_TEMPLATE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len_32 += (unsigned)len; + state->large_len |= (len>=16) | (state->total_len_32>=16); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (unsigned)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const U32* p32 = state->mem32; + state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; + state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; + state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do { + v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; + v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; + v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; + v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE_TEMPLATE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) +{ + const BYTE * p = (const BYTE*)state->mem32; + const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; + U32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } else { + h32 = state->v3 /* == seed */ + PRIME32_5; + } + + h32 += state->total_len_32; + + while (p+4<=bEnd) { + h32 += XXH_readLE32(p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p<bEnd) { + h32 += (*p) * PRIME32_5; + h32 = XXH_rotl32(h32, 11) * PRIME32_1; + p++; + } + + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_digest_endian(state_in, XXH_littleEndian); + else + return XXH32_digest_endian(state_in, XXH_bigEndian); +} + + + +/* **** XXH64 **** */ + +FORCE_INLINE_TEMPLATE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + if (input != NULL) { + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + } + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); + state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); + state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); + state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do { + v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; + v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; + v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; + v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE_TEMPLATE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) +{ + const BYTE * p = (const BYTE*)state->mem64; + const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; + U64 h64; + + if (state->total_len >= 32) { + U64 const v1 = state->v1; + U64 const v2 = state->v2; + U64 const v3 = state->v3; + U64 const v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + } else { + h64 = state->v3 + PRIME64_5; + } + + h64 += (U64) state->total_len; + + while (p+8<=bEnd) { + U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) { + h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p<bEnd) { + h64 ^= (*p) * PRIME64_5; + h64 = XXH_rotl64(h64, 11) * PRIME64_1; + p++; + } + + h64 ^= h64 >> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_digest_endian(state_in, XXH_littleEndian); + else + return XXH64_digest_endian(state_in, XXH_bigEndian); +} + + +/* ************************** +* Canonical representation +****************************/ + +/*! Default XXH result types are basic unsigned 32 and 64 bits. +* The canonical representation follows human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. +*/ + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + ZSTD_memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + ZSTD_memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} diff --git a/src/bled/xxhash.h b/src/bled/xxhash.h new file mode 100644 index 00000000000..8865cfd819e --- /dev/null +++ b/src/bled/xxhash.h @@ -0,0 +1,276 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - xxHash source repository : https://github.com/Cyan4973/xxHash + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +A 64-bits version, named XXH64, is available since r35. +It offers much better speed, but for 64-bits applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + + +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + + +/* **************************** +* Definitions +******************************/ +#include "zstd_deps.h" +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/* **************************** +* API modifier +******************************/ +/** XXH_PRIVATE_API +* This is useful if you want to include xxhash functions in `static` mode +* in order to inline them, and remove their symbol from the public list. +* Methodology : +* #define XXH_PRIVATE_API +* #include "xxhash.h" +* `xxhash.c` is automatically included. +* It's not useful to compile and link it as a separate module anymore. +*/ +#ifdef XXH_PRIVATE_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else +# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ +# endif +#else +# define XXH_PUBLIC_API /* do nothing */ +#endif /* XXH_PRIVATE_API */ + +/*!XXH_NAMESPACE, aka Namespace Emulation : + +If you want to include _and expose_ xxHash functions from within your own library, +but also want to avoid symbol collisions with another library which also includes xxHash, + +you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library +with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). + +Note that no change is required within the calling program as long as it includes `xxhash.h` : +regular symbol name will be automatically translated by this header. +*/ +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 6 +#define XXH_VERSION_RELEASE 2 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/* **************************** +* Simple Hash Functions +******************************/ +typedef unsigned int XXH32_hash_t; +typedef unsigned long long XXH64_hash_t; + +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); +XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); + +/*! +XXH32() : + Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s +XXH64() : + Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + "seed" can be used to alter the result predictably. + This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark). +*/ + + +/* **************************** +* Streaming Hash Functions +******************************/ +typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ + +/*! State allocation, compatible with dynamic libraries */ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + + +/* hash streaming */ + +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/* +These functions generate the xxHash of an input provided in multiple segments. +Note that, for small input, they are slower than single-call functions, due to state management. +For small input, prefer `XXH32()` and `XXH64()` . + +XXH state must first be allocated, using XXH*_createState() . + +Start a new hash by initializing state with a seed, using XXH*_reset(). + +Then, feed the hash state by calling XXH*_update() as many times as necessary. +Obviously, input must be allocated and read accessible. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + +Finally, a hash value can be produced anytime, by using XXH*_digest(). +This function returns the nn-bits hash as an int or long long. + +It's still possible to continue inserting input into the hash state after a digest, +and generate some new hashes later on, by calling again XXH*_digest(). + +When done, free XXH state space if it was allocated dynamically. +*/ + + +/* ************************** +* Utils +****************************/ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */ +# define restrict /* disable restrict */ +#endif + +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state); + + +/* ************************** +* Canonical representation +****************************/ +/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. +* The canonical representation uses human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. +*/ +typedef struct { unsigned char digest[4]; } XXH32_canonical_t; +typedef struct { unsigned char digest[8]; } XXH64_canonical_t; + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + +#endif /* XXHASH_H_5627135585666179 */ + + + +/* ================================================================================================ + This section contains definitions which are not guaranteed to remain stable. + They may change in future versions, becoming incompatible with a different version of the library. + They shall only be used with static linking. + Never use these definitions in association with dynamic linking ! +=================================================================================================== */ +#if !defined(XXH_STATIC_H_3543687687345) +#define XXH_STATIC_H_3543687687345 + +/* These definitions are only meant to allow allocation of XXH state + statically, on stack, or in a struct for example. + Do not use members directly. */ + + struct XXH32_state_s { + unsigned total_len_32; + unsigned large_len; + unsigned v1; + unsigned v2; + unsigned v3; + unsigned v4; + unsigned mem32[4]; /* buffer defined as U32 for alignment */ + unsigned memsize; + unsigned reserved; /* never read nor write, will be removed in a future version */ + }; /* typedef'd to XXH32_state_t */ + + struct XXH64_state_s { + unsigned long long total_len; + unsigned long long v1; + unsigned long long v2; + unsigned long long v3; + unsigned long long v4; + unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ + unsigned memsize; + unsigned reserved[2]; /* never read nor write, will be removed in a future version */ + }; /* typedef'd to XXH64_state_t */ + + +# ifdef XXH_PRIVATE_API +# include "xxhash.c" /* include xxhash functions as `static`, for inlining */ +# endif + +#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */ + + diff --git a/src/bled/xz_config.h b/src/bled/xz_config.h index 682ec553c72..05d90af95b3 100644 --- a/src/bled/xz_config.h +++ b/src/bled/xz_config.h @@ -18,7 +18,6 @@ /* #define XZ_DEC_ARMTHUMB */ /* #define XZ_DEC_SPARC */ -#include <assert.h> #include <stdbool.h> #include <stdlib.h> #include <string.h> diff --git a/src/bled/xz_dec_lzma2.c b/src/bled/xz_dec_lzma2.c index ed519d6d8ed..a6911db837b 100644 --- a/src/bled/xz_dec_lzma2.c +++ b/src/bled/xz_dec_lzma2.c @@ -10,6 +10,7 @@ #include "xz_private.h" #include "xz_lzma2.h" +#include <assert.h> /* * Range decoder initialization eats the first five bytes of each LZMA chunk. diff --git a/src/bled/zstd.h b/src/bled/zstd.h new file mode 100644 index 00000000000..858d434a0ac --- /dev/null +++ b/src/bled/zstd.h @@ -0,0 +1,148 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 + +/* ====== Dependency ======*/ +#include <limits.h> /* INT_MAX */ +#include <stddef.h> /* size_t */ +#include "zstd_config.h" + + +/* ===== ZSTDLIB_API : control library symbols visibility ===== */ +#define ZSTDLIB_VISIBILITY +#define ZSTDLIB_API ZSTDLIB_VISIBILITY + +/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ +#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ +#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 + +#define ZSTD_BLOCKSIZELOG_MAX 17 +#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) + +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); +ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +typedef struct ZSTD_CCtx_s ZSTD_CCtx; +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ + +typedef enum { + ZSTD_reset_session_only = 1, + ZSTD_reset_parameters = 2, + ZSTD_reset_session_and_parameters = 3 +} ZSTD_ResetDirective; +ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + +typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */ + +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); + +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + +typedef struct ZSTD_DDict_s ZSTD_DDict; +ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); +ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); + +#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ +#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ +#define ZSTD_SKIPPABLEHEADERSIZE 8 + +/* compression parameter bounds */ +#define ZSTD_WINDOWLOG_MAX_32 30 +#define ZSTD_WINDOWLOG_MAX_64 31 +#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) + + +#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame + */ + + + +typedef enum { + ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. + * Useful to save 4 bytes per generated frame. + * Decoder cannot recognise automatically this format, requiring this instruction. */ +} ZSTD_format_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ + ZSTD_d_validateChecksum = 0, + ZSTD_d_ignoreChecksum = 1 +} ZSTD_forceIgnoreChecksum_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_refMultipleDDicts */ + ZSTD_rmd_refSingleDDict = 0, + ZSTD_rmd_refMultipleDDicts = 1 +} ZSTD_refMultipleDDicts_e; + +ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); + +ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); +ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); + +/*===== Buffer-less streaming decompression functions =====*/ +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; + unsigned checksumFlag; +} ZSTD_frameHeader; + +ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ +ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); +ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + +ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ + diff --git a/src/bled/zstd_compiler.h b/src/bled/zstd_compiler.h new file mode 100644 index 00000000000..0064dac07f0 --- /dev/null +++ b/src/bled/zstd_compiler.h @@ -0,0 +1,242 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPILER_H +#define ZSTD_COMPILER_H + +/*-******************************************************* +* Compiler specifics +*********************************************************/ +/* force inlining */ + +#if !defined(ZSTD_NO_INLINE) +#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# define INLINE_KEYWORD inline +#else +# define INLINE_KEYWORD +#endif + +#if defined(__GNUC__) || defined(__ICCARM__) +# define FORCE_INLINE_ATTR __attribute__((always_inline)) +#elif defined(_MSC_VER) +# define FORCE_INLINE_ATTR __forceinline +#else +# define FORCE_INLINE_ATTR +#endif + +#else + +#define INLINE_KEYWORD +#define FORCE_INLINE_ATTR + +#endif + +/** + On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). + This explictly marks such functions as __cdecl so that the code will still compile + if a CC other than __cdecl has been made the default. +*/ +#if defined(_MSC_VER) +# define WIN_CDECL __cdecl +#else +# define WIN_CDECL +#endif + +/** + * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant + * parameters. They must be inlined for the compiler to eliminate the constant + * branches. + */ +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR +/** + * HINT_INLINE is used to help the compiler generate better code. It is *not* + * used for "templates", so it can be tweaked based on the compilers + * performance. + * + * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the + * always_inline attribute. + * + * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline + * attribute. + */ +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 +# define HINT_INLINE static INLINE_KEYWORD +#else +# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR +#endif + +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + +/* force no inlining */ +#ifdef _MSC_VER +# define FORCE_NOINLINE static __declspec(noinline) +#else +# if defined(__GNUC__) || defined(__ICCARM__) +# define FORCE_NOINLINE static __attribute__((__noinline__)) +# else +# define FORCE_NOINLINE static +# endif +#endif + + +/* target attribute */ +#ifndef __has_attribute + #define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */ +#endif +#if defined(__GNUC__) || defined(__ICCARM__) +# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) +#else +# define TARGET_ATTRIBUTE(target) +#endif + +/* Enable runtime BMI2 dispatch based on the CPU. + * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. + */ +#ifndef DYNAMIC_BMI2 + #if ((defined(__clang__) && __has_attribute(__target__)) \ + || (defined(__GNUC__) \ + && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ + && (defined(__x86_64__) || defined(_M_X64)) \ + && !defined(__BMI2__) + # define DYNAMIC_BMI2 1 + #else + # define DYNAMIC_BMI2 0 + #endif +#endif + +/* prefetch + * can be disabled, by declaring NO_PREFETCH build macro */ +#if defined(NO_PREFETCH) +# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ +# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) +# elif defined(__aarch64__) +# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) +# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) +# else +# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* NO_PREFETCH */ + +#define CACHELINE_SIZE 64 + +#define PREFETCH_AREA(p, s) { \ + const char* const _ptr = (const char*)(p); \ + size_t const _size = (size_t)(s); \ + size_t _pos; \ + for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ + PREFETCH_L2(_ptr + _pos); \ + } \ +} + +/* vectorization + * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax, + * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */ +#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__) +# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) +# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +# else +# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")") +# endif +#else +# define DONT_VECTORIZE +#endif + +/* Tell the compiler that a branch is likely or unlikely. + * Only use these macros if it causes the compiler to generate better code. + * If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc + * and clang, please do. + */ +#if defined(__GNUC__) +#define LIKELY(x) (__builtin_expect((x), 1)) +#define UNLIKELY(x) (__builtin_expect((x), 0)) +#else +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#endif + +/* disable warnings */ +#ifdef _MSC_VER /* Visual Studio */ +# include <intrin.h> /* For Visual 2005 */ +# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +# pragma warning(disable : 4324) /* disable: C4324: padded structure */ +#endif + +/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/ +#ifndef STATIC_BMI2 +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) +# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 +# define STATIC_BMI2 1 +# endif +# endif +#endif + +#ifndef STATIC_BMI2 + #define STATIC_BMI2 0 +#endif + +/* compile time determination of SIMD support */ + +/* compat. with non-clang compilers */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define ZSTD_HAS_C_ATTRIBUTE(x) 0 +#endif + +/* Only use C++ attributes in C++. Some compilers report support for C++ + * attributes when compiling with C. + */ +#define ZSTD_HAS_CPP_ATTRIBUTE(x) 0 + +/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute. + * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough + * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * - Else: __attribute__((__fallthrough__)) + */ +#ifndef ZSTD_FALLTHROUGH +# if ZSTD_HAS_C_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif __has_attribute(__fallthrough__) +# define ZSTD_FALLTHROUGH __attribute__((__fallthrough__)) +# else +# define ZSTD_FALLTHROUGH +# endif +#endif + +#endif /* ZSTD_COMPILER_H */ diff --git a/src/bled/zstd_config.h b/src/bled/zstd_config.h new file mode 100644 index 00000000000..4792e1e2460 --- /dev/null +++ b/src/bled/zstd_config.h @@ -0,0 +1,49 @@ +#ifndef ZSTD_CONFIG +#define ZSTD_CONFIG + +#define ZSTD_DEBUGLEVEL 0 +#define ZSTD_LEGACY_SUPPORT 0 +#define ZSTD_LIB_DEPRECATED 0 +#define ZSTD_NO_UNUSED_FUNCTIONS 1 +#define ZSTD_STRIP_ERROR_STRINGS 0 +#define ZSTD_TRACE 0 +#define ZSTD_DECOMPRESS_DICTIONARY 0 +#define ZSTD_DECOMPRESS_MULTIFRAME 0 + +#if CONFIG_FEATURE_ZSTD_SMALL >= 9 +#define ZSTD_NO_INLINE 1 +#endif + +#if CONFIG_FEATURE_ZSTD_SMALL >= 7 +#define HUF_FORCE_DECOMPRESS_X1 1 +#define ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT 1 +#elif CONFIG_FEATURE_ZSTD_SMALL >= 5 +#define HUF_FORCE_DECOMPRESS_X1 1 +#endif + +#if CONFIG_FEATURE_ZSTD_SMALL <= 7 +/* doesnt blow up code too much, -O3 is horrible */ +#ifdef __GNUC__ +#pragma GCC optimize ("O2") +#endif +#endif + +#if CONFIG_FEATURE_ZSTD_SMALL > 0 +/* no dynamic detection of bmi2 instruction, + * prefer using CFLAGS setting to -march=haswell or similar */ +# if !defined(__BMI2__) +# define DYNAMIC_BMI2 0 +# endif +#endif + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-function" +#endif + +/* Include zstd_deps.h first with all the options we need enabled. */ +#define ZSTD_DEPS_NEED_MALLOC +#define ZSTD_DEPS_NEED_MATH64 +#define ZSTD_DEPS_NEED_STDINT + +#endif diff --git a/src/bled/zstd_cpu.h b/src/bled/zstd_cpu.h new file mode 100644 index 00000000000..6866f60f464 --- /dev/null +++ b/src/bled/zstd_cpu.h @@ -0,0 +1,213 @@ +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMMON_CPU_H +#define ZSTD_COMMON_CPU_H + +/** + * Implementation taken from folly/CpuId.h + * https://github.com/facebook/folly/blob/master/folly/CpuId.h + */ + +#include "zstd_mem.h" + +#ifdef _MSC_VER +#include <intrin.h> +#endif + +typedef struct { + U32 f1c; + U32 f1d; + U32 f7b; + U32 f7c; +} ZSTD_cpuid_t; + +MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { + U32 f1c = 0; + U32 f1d = 0; + U32 f7b = 0; + U32 f7c = 0; +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) + int reg[4]; + __cpuid((int*)reg, 0); + { + int const n = reg[0]; + if (n >= 1) { + __cpuid((int*)reg, 1); + f1c = (U32)reg[2]; + f1d = (U32)reg[3]; + } + if (n >= 7) { + __cpuidex((int*)reg, 7, 0); + f7b = (U32)reg[1]; + f7c = (U32)reg[2]; + } + } +#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) + /* The following block like the normal cpuid branch below, but gcc + * reserves ebx for use of its pic register so we must specially + * handle the save and restore to avoid clobbering the register + */ + U32 n; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(n) + : "a"(0) + : "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1)); + } + if (n >= 7) { + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "movl %%ebx, %%eax\n\t" + "popl %%ebx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + U32 n; + __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); + } + if (n >= 7) { + U32 f7a; + __asm__("cpuid" + : "=a"(f7a), "=b"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#endif + { + ZSTD_cpuid_t cpuid; + cpuid.f1c = f1c; + cpuid.f1d = f1d; + cpuid.f7b = f7b; + cpuid.f7c = f7c; + return cpuid; + } +} + +#define X(name, r, bit) \ + MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ + return ((cpuid.r) & (1U << bit)) != 0; \ + } + +/* cpuid(1): Processor Info and Feature Bits. */ +#define C(name, bit) X(name, f1c, bit) + C(sse3, 0) + C(pclmuldq, 1) + C(dtes64, 2) + C(monitor, 3) + C(dscpl, 4) + C(vmx, 5) + C(smx, 6) + C(eist, 7) + C(tm2, 8) + C(ssse3, 9) + C(cnxtid, 10) + C(fma, 12) + C(cx16, 13) + C(xtpr, 14) + C(pdcm, 15) + C(pcid, 17) + C(dca, 18) + C(sse41, 19) + C(sse42, 20) + C(x2apic, 21) + C(movbe, 22) + C(popcnt, 23) + C(tscdeadline, 24) + C(aes, 25) + C(xsave, 26) + C(osxsave, 27) + C(avx, 28) + C(f16c, 29) + C(rdrand, 30) +#undef C +#define D(name, bit) X(name, f1d, bit) + D(fpu, 0) + D(vme, 1) + D(de, 2) + D(pse, 3) + D(tsc, 4) + D(msr, 5) + D(pae, 6) + D(mce, 7) + D(cx8, 8) + D(apic, 9) + D(sep, 11) + D(mtrr, 12) + D(pge, 13) + D(mca, 14) + D(cmov, 15) + D(pat, 16) + D(pse36, 17) + D(psn, 18) + D(clfsh, 19) + D(ds, 21) + D(acpi, 22) + D(mmx, 23) + D(fxsr, 24) + D(sse, 25) + D(sse2, 26) + D(ss, 27) + D(htt, 28) + D(tm, 29) + D(pbe, 31) +#undef D + +/* cpuid(7): Extended Features. */ +#define B(name, bit) X(name, f7b, bit) + B(bmi1, 3) + B(hle, 4) + B(avx2, 5) + B(smep, 7) + B(bmi2, 8) + B(erms, 9) + B(invpcid, 10) + B(rtm, 11) + B(mpx, 14) + B(avx512f, 16) + B(avx512dq, 17) + B(rdseed, 18) + B(adx, 19) + B(smap, 20) + B(avx512ifma, 21) + B(pcommit, 22) + B(clflushopt, 23) + B(clwb, 24) + B(avx512pf, 26) + B(avx512er, 27) + B(avx512cd, 28) + B(sha, 29) + B(avx512bw, 30) + B(avx512vl, 31) +#undef B +#define C(name, bit) X(name, f7c, bit) + C(prefetchwt1, 0) + C(avx512vbmi, 1) +#undef C + +#undef X + +#endif /* ZSTD_COMMON_CPU_H */ diff --git a/src/bled/zstd_ddict.h b/src/bled/zstd_ddict.h new file mode 100644 index 00000000000..ca4dccbe5a2 --- /dev/null +++ b/src/bled/zstd_ddict.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DDICT_H +#define ZSTD_DDICT_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "zstd_deps.h" /* size_t */ +#include "zstd.h" /* ZSTD_DDict, and several public functions */ + + +/*-******************************************************* + * Interface + *********************************************************/ + +/* note: several prototypes are already published in `zstd.h` : + * ZSTD_createDDict() + * ZSTD_createDDict_byReference() + * ZSTD_createDDict_advanced() + * ZSTD_freeDDict() + * ZSTD_initStaticDDict() + * ZSTD_sizeof_DDict() + * ZSTD_estimateDDictSize() + * ZSTD_getDictID_fromDict() + */ + +MEM_STATIC const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); +MEM_STATIC size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + +MEM_STATIC void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +#if ZSTD_DECOMPRESS_DICTIONARY == 0 +MEM_STATIC void ZSTD_clearDict(ZSTD_DCtx* dctx) { (void)dctx; } +MEM_STATIC size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) { (void)dctx; (void)ddict; return 0; } +MEM_STATIC size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output) { (void)zds; (void)output; return 0; } +MEM_STATIC ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) { (void)dctx; return NULL; } +#endif + +#endif /* ZSTD_DDICT_H */ diff --git a/src/bled/zstd_decompress.c b/src/bled/zstd_decompress.c new file mode 100644 index 00000000000..98f2afb6470 --- /dev/null +++ b/src/bled/zstd_decompress.c @@ -0,0 +1,1248 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ + +/*! +* LEGACY_SUPPORT : +* if set to 1+, ZSTD_decompress() can decode older formats (v0.1+) +*/ + +/*! + * MAXWINDOWSIZE_DEFAULT : + * maximum window size accepted by DStream __by default__. + * Frames requiring more memory will be rejected. + * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize(). + */ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1) +#endif + +/*! + * NO_FORWARD_PROGRESS_MAX : + * maximum allowed nb of calls to ZSTD_decompressStream() + * without any forward progress + * (defined as: no byte read from input, and no byte flushed to output) + * before triggering an error. + */ +#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX +# define ZSTD_NO_FORWARD_PROGRESS_MAX 16 +#endif + + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "zstd_cpu.h" /* bmi2 */ +#include "zstd_mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */ +#include "zstd_internal.h" /* blockProperties_t */ +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ + +/*-************************************************************* +* Context management +***************************************************************/ +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support sizeof NULL */ + return sizeof(*dctx) +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + + ZSTD_sizeof_DDict(dctx->ddictLocal) +#endif + + dctx->inBuffSize + dctx->outBuffSize; +} + +static size_t ZSTD_startingInputLength(ZSTD_format_e format) +{ + size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format); + /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ + assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) ); + return startingInputLength; +} + +static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) +{ + assert(dctx->streamStage == zdss_init); + dctx->format = ZSTD_f_zstd1; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + // dctx->outBufferMode = ZSTD_bm_buffered; REMOVED + dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict; +#endif +} + +static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) +{ + // dctx->staticSize = 0; REMOVED +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + dctx->ddict = NULL; + dctx->ddictLocal = NULL; + dctx->dictEnd = NULL; + dctx->ddictIsCold = 0; + dctx->dictUses = ZSTD_dont_use; + dctx->ddictSet = NULL; +#endif + dctx->inBuff = NULL; + dctx->inBuffSize = 0; + dctx->outBuffSize = 0; + dctx->streamStage = zdss_init; + dctx->noForwardProgress = 0; + dctx->oversizedDuration = 0; +#if DYNAMIC_BMI2 + dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); +#endif + ZSTD_DCtx_resetParameters(dctx); +} + +static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) { + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem); + if (!dctx) return NULL; + ZSTD_initDCtx_internal(dctx); + return dctx; + } +} + +size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support free on NULL */ + RETURN_ERROR_IF(ZSTD_DCtx_get_staticSize(zds), memory_allocation, "not compatible with static DCtx"); + { ZSTD_customMem const cMem = ZSTD_defaultCMem; +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + ZSTD_clearDict(dctx); +#endif + ZSTD_customFree(dctx->inBuff, cMem); + dctx->inBuff = NULL; +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + if (dctx->ddictSet) { + ZSTD_freeDDictHashSet(dctx->ddictSet, cMem); + dctx->ddictSet = NULL; + } +#endif + ZSTD_customFree(dctx, cMem); + return 0; + } +} + +/* no longer useful */ +void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx); + ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ +} + +/*-************************************************************* + * Frame header decoding + ***************************************************************/ + +/** ZSTD_frameHeaderSize_internal() : + * srcSize must be large enough to reach header size fields. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. + * @return : size of the Frame Header + * or an error code, which can be tested with ZSTD_isError() */ +static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format) +{ + size_t const minInputSize = ZSTD_startingInputLength(format); + RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, ""); + + { BYTE const fhd = ((const BYTE*)src)[minInputSize-1]; + U32 const dictID= fhd & 3; + U32 const singleSegment = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return minInputSize + !singleSegment + + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + + (singleSegment && !fcsId); + } +} + +/** ZSTD_getFrameHeader_advanced() : + * decode Frame Header, or require larger `srcSize`. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) +{ + const BYTE* ip = (const BYTE*)src; + size_t const minInputSize = ZSTD_startingInputLength(format); + + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ + if (srcSize < minInputSize) return minInputSize; + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter"); + + if ( (format != ZSTD_f_zstd1_magicless) + && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame */ + if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) + return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); + zfhPtr->frameType = ZSTD_skippableFrame; + return 0; + } + RETURN_ERROR(prefix_unknown, ""); + } + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format); + if (srcSize < fhsize) return fhsize; + zfhPtr->headerSize = (U32)fhsize; + } + + { BYTE const fhdByte = ip[minInputSize-1]; + size_t pos = minInputSize; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const singleSegment = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U64 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN; + RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported, + "reserved bits, must be zero"); + + if (!singleSegment) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, ""); + windowSize = (1ULL << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); + } + switch(dictIDSizeCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(fcsID) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : if (singleSegment) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (singleSegment) windowSize = frameContentSize; + + zfhPtr->frameType = ZSTD_frame; + zfhPtr->frameContentSize = frameContentSize; + zfhPtr->windowSize = windowSize; + zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + zfhPtr->dictID = dictID; + zfhPtr->checksumFlag = checksumFlag; + } + return 0; +} + +/** ZSTD_getFrameHeader() : + * decode Frame Header, or require larger `srcSize`. + * note : this function does not consume input, it only reads it. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) +{ + return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); +} + +/** ZSTD_getFrameContentSize() : + * compatible with legacy mode + * @return : decompressed size of the single frame pointed to be `src` if known, otherwise + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ +unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) +{ + { ZSTD_frameHeader zfh; + if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) + return ZSTD_CONTENTSIZE_ERROR; + if (zfh.frameType == ZSTD_skippableFrame) { + return 0; + } else { + return zfh.frameContentSize; + } } +} + + +/** ZSTD_decodeFrameHeader() : + * `headerSize` must be the size provided by ZSTD_frameHeaderSize(). + * If multiple DDict references are enabled, also will choose the correct DDict to use. + * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) +{ + size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format); + U32 dictID = 0; + if (ZSTD_isError(result)) return result; /* invalid header */ + RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small"); + +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + /* Reference DDict requested by frame if dctx references multiple ddicts */ + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) { + ZSTD_DCtx_selectFrameDDict(dctx); + } + dictID = dctx->dictID; +#endif + + /* Skip the dictID check in fuzzing mode, because it makes the search + * harder. + */ + + RETURN_ERROR_IF(dctx->fParams.dictID && (dictID != dctx->fParams.dictID), + dictionary_wrong, ""); + + dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0; + if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0); + dctx->processedCSize += headerSize; + return 0; +} + +static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) +{ + ZSTD_frameSizeInfo frameSizeInfo; + frameSizeInfo.compressedSize = ret; + frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; + return frameSizeInfo; +} + +/*-************************************************************* + * Frame decoding + ***************************************************************/ + +static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_copyRawBlock"); + RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (srcSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memcpy(dst, src, srcSize); + return srcSize; +} + +static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, + BYTE b, + size_t regenSize) +{ + RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (regenSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memset(dst, b, regenSize); + return regenSize; +} + +static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming) +{ + (void)dctx; + (void)uncompressedSize; + (void)compressedSize; + (void)streaming; +} + + +/*! ZSTD_decompressFrame() : + * @dctx must be properly initialized + * will update *srcPtr and *srcSizePtr, + * to make *srcPtr progress by one frame. */ +static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void** srcPtr, size_t *srcSizePtr) +{ + const BYTE* const istart = (const BYTE*)(*srcPtr); + const BYTE* ip = istart; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart; + BYTE* op = ostart; + size_t remainingSrcSize = *srcSizePtr; + + DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr); + + /* check */ + RETURN_ERROR_IF( + remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal( + ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format); + if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; + RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , ""); + ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + size_t decodedSize; + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSrcSize -= ZSTD_blockHeaderSize; + RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); + + switch(blockProperties.blockType) + { + case bt_compressed: + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1); + break; + case bt_raw : + decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); + break; + case bt_rle : + decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize); + break; + case bt_reserved : + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + + if (ZSTD_isError(decodedSize)) return decodedSize; + if (dctx->validateChecksum) + XXH64_update(&dctx->xxhState, op, decodedSize); + if (decodedSize != 0) + op += decodedSize; + assert(ip != NULL); + ip += cBlockSize; + remainingSrcSize -= cBlockSize; + if (blockProperties.lastBlock) break; + } + + if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize, + corruption_detected, ""); + } + if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ + RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, ""); + if (!dctx->forceIgnoreChecksum) { + U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); + U32 checkRead; + checkRead = MEM_readLE32(ip); + RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, ""); + } + ip += 4; + remainingSrcSize -= 4; + } + ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); + /* Allow caller to get size read */ + *srcPtr = ip; + *srcSizePtr = remainingSrcSize; + return (size_t)(op-ostart); +} + +/*-************************************** +* Advanced Streaming Decompression API +* Bufferless and synchronous +****************************************/ +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } + +/** + * Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed, + * we allow taking a partial block as the input. Currently only raw uncompressed blocks can + * be streamed. + * + * For blocks that can be streamed, this allows us to reduce the latency until we produce + * output, and avoid copying the input. + * + * @param inputSize - The total amount of input that the caller currently has. + */ +static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) { + if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock)) + return dctx->expected; + if (dctx->bType != bt_raw) + return dctx->expected; + return MIN(MAX(inputSize, 1), dctx->expected); +} + +ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { + switch(dctx->stage) + { + default: /* should not happen */ + assert(0); + ZSTD_FALLTHROUGH; + case ZSTDds_getFrameHeaderSize: + ZSTD_FALLTHROUGH; + case ZSTDds_decodeFrameHeader: + return ZSTDnit_frameHeader; + case ZSTDds_decodeBlockHeader: + return ZSTDnit_blockHeader; + case ZSTDds_decompressBlock: + return ZSTDnit_block; + case ZSTDds_decompressLastBlock: + return ZSTDnit_lastBlock; + case ZSTDds_checkChecksum: + return ZSTDnit_checksum; + case ZSTDds_decodeSkippableHeader: + ZSTD_FALLTHROUGH; + case ZSTDds_skipFrame: + return ZSTDnit_skippableFrame; + } +} + +static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } + +/** ZSTD_decompressContinue() : + * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress()) + * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize); + /* Sanity check */ + RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed"); + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + dctx->processedCSize += srcSize; + + switch (dctx->stage) + { + case ZSTDds_getFrameHeaderSize : + assert(src != NULL); + if (dctx->format == ZSTD_f_zstd1) { /* allows header */ + assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } } + dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format); + if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = dctx->headerSize - srcSize; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; + + case ZSTDds_decodeFrameHeader: + assert(src != NULL); + ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), ""); + dctx->expected = ZSTD_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + + case ZSTDds_decodeBlockHeader: + { blockProperties_t bp; + size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum"); + dctx->expected = cBlockSize; + dctx->bType = bp.blockType; + dctx->rleSize = bp.origSize; + if (cBlockSize) { + dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; + return 0; + } + /* empty block */ + if (bp.lastBlock) { + if (dctx->fParams.checksumFlag) { + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* end of frame */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */ + dctx->stage = ZSTDds_decodeBlockHeader; + } + return 0; + } + + case ZSTDds_decompressLastBlock: + case ZSTDds_decompressBlock: + DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock"); + { size_t rSize; + switch(dctx->bType) + { + case bt_compressed: + DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_raw : + assert(srcSize <= dctx->expected); + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); + FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed"); + assert(rSize == srcSize); + dctx->expected -= rSize; + break; + case bt_rle : + rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_reserved : /* should never happen */ + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + FORWARD_IF_ERROR(rSize, ""); + RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum"); + DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); + dctx->decodedSize += rSize; + if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize); + dctx->previousDstEnd = (char*)dst + rSize; + + /* Stay on the same stage until we are finished streaming the block. */ + if (dctx->expected > 0) { + return rSize; + } + + if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ + DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize); + RETURN_ERROR_IF( + dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && dctx->decodedSize != dctx->fParams.frameContentSize, + corruption_detected, ""); + if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; /* ends here */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->stage = ZSTDds_decodeBlockHeader; + dctx->expected = ZSTD_blockHeaderSize; + } + return rSize; + } + + case ZSTDds_checkChecksum: + assert(srcSize == 4); /* guaranteed by dctx->expected */ + { + if (dctx->validateChecksum) { + U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); + U32 const check32 = MEM_readLE32(src); + DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); + RETURN_ERROR_IF(check32 != h32, checksum_wrong, ""); + } + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + + case ZSTDds_decodeSkippableHeader: + assert(src != NULL); + assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ + dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ + dctx->stage = ZSTDds_skipFrame; + return 0; + + case ZSTDds_skipFrame: + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + } +} + + +static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dict; + dctx->previousDstEnd = (const char*)dict + dictSize; + return 0; +} + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of entropy tables read */ +size_t +ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + + RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small"); + assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */ + dictPtr += 8; /* skip header = magic + dictID */ + + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable)); + ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE); + { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */ + size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable); +#ifdef HUF_FORCE_DECOMPRESS_X1 + /* in minimal huffman, we always use X1 variants */ + size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, + dictPtr, dictEnd - dictPtr, + workspace, workspaceSize); +#else + size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, + dictPtr, (size_t)(dictEnd - dictPtr), + workspace, workspaceSize); +#endif + RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); + dictPtr += hSize; + } + + { short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff, offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->OFTable, + offcodeNCount, offcodeMaxValue, + OF_base, OF_bits, + offcodeLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */0); + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->MLTable, + matchlengthNCount, matchlengthMaxValue, + ML_base, ML_bits, + matchlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->LLTable, + litlengthNCount, litlengthMaxValue, + LL_base, LL_bits, + litlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += litlengthHeaderSize; + } + + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); + { int i; + size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); + for (i=0; i<3; i++) { + U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; + RETURN_ERROR_IF(rep==0 || rep > dictContentSize, + dictionary_corrupted, ""); + entropy->rep[i] = rep; + } } + + return (size_t)(dictPtr - (const BYTE*)dict); +} + +size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) +{ + assert(dctx != NULL); + dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ + dctx->stage = ZSTDds_getFrameHeaderSize; + dctx->processedCSize = 0; + dctx->decodedSize = 0; + dctx->previousDstEnd = NULL; + dctx->prefixStart = NULL; + dctx->virtualStart = NULL; + dctx->dictEnd = NULL; + dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + dctx->litEntropy = dctx->fseEntropy = 0; +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + dctx->dictID = 0; +#endif + dctx->bType = bt_reserved; + ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); + ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->entropy.LLTable; + dctx->MLTptr = dctx->entropy.MLTable; + dctx->OFTptr = dctx->entropy.OFTable; + dctx->HUFptr = dctx->entropy.hufTable; + return 0; +} + +size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + if (dict && dictSize) + RETURN_ERROR_IF( + ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)), + dictionary_corrupted, ""); +#endif + (void)dict; (void)dictSize; + return 0; +} + + +/* ====== ZSTD_DDict ====== */ + +size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict"); + assert(dctx != NULL); + (void)ddict; +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + if (ddict) { + const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict); + size_t const dictSize = ZSTD_DDict_dictSize(ddict); + const void* const dictEnd = dictStart + dictSize; + dctx->ddictIsCold = (dctx->dictEnd != dictEnd); + DEBUGLOG(4, "DDict is %s", + dctx->ddictIsCold ? "~cold~" : "hot!"); + } +#endif + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + if (ddict) { /* NULL ddict is equivalent to no dictionary */ + ZSTD_copyDDictParameters(dctx, ddict); + } +#endif + return 0; +} + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0; + return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); +} + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompress frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary (most common case). + * - The frame was built with dictID intentionally removed. + * Needed dictionary is a hidden information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, frame header could not be decoded. + * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use + * ZSTD_getFrameHeader(), which will provide a more precise error code. */ +unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) +{ + ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 }; + size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); + if (ZSTD_isError(hError)) return 0; + return zfp.dictID; +} + +/*===================================== +* Streaming decompression +*====================================*/ + +ZSTD_DStream* ZSTD_createDStream(void) +{ + DEBUGLOG(3, "ZSTD_createDStream"); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + return ZSTD_freeDCtx(zds); +} + + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } +size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } + + +/* note : this variant can't fail */ +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + DEBUGLOG(4, "ZSTD_initDStream"); + return ZSTD_initDStream_usingDDict(zds, NULL); +} + +/* ZSTD_initDStream_usingDDict() : + * ddict will just be referenced, and must outlive decompression session + * this function cannot fail */ +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) +{ + FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); + return ZSTD_startingInputLength(dctx->format); +} + +/* ZSTD_resetDStream() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_resetDStream(ZSTD_DStream* dctx) +{ + FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); + return ZSTD_startingInputLength(dctx->format); +} + + + +size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + dctx->streamStage = zdss_init; + dctx->noForwardProgress = 0; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + ZSTD_DCtx_resetParameters(dctx); + } + return 0; +} + +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2); + unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); + size_t const minRBSize = (size_t) neededSize; + RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, + frameParameter_windowTooLarge, ""); + return minRBSize; +} + + +/* ***** Decompression ***** */ + +static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR; +} + +static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize)) + zds->oversizedDuration++; + else + zds->oversizedDuration = 0; +} + +static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds) +{ + return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream() + * and updates the stage and the output buffer state. This call is extracted so it can be + * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode. + * NOTE: You must break after calling this function since the streamStage is modified. + */ +static size_t ZSTD_decompressContinueStream( + ZSTD_DStream* zds, char** op, char* oend, + void const* src, size_t srcSize) { + int const isSkipFrame = ZSTD_isSkipFrame(zds); + if (ZSTD_DCtx_get_outBufferMode(zds) == ZSTD_bm_buffered) { + size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart; + size_t const decodedSize = ZSTD_decompressContinue(zds, + zds->outBuff + zds->outStart, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + if (!decodedSize && !isSkipFrame) { + zds->streamStage = zdss_read; + } else { + zds->outEnd = zds->outStart + decodedSize; + zds->streamStage = zdss_flush; + } + } else { + /* Write directly into the output buffer */ + size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op); + size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + *op += decodedSize; + /* Flushing is not needed. */ + zds->streamStage = zdss_read; + assert(*op <= oend); + assert(ZSTD_DCtx_get_outBufferMode(zds) == ZSTD_bm_stable); + } + return 0; +} + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + const char* const src = (const char*)input->src; + const char* const istart = input->pos != 0 ? src + input->pos : src; + const char* const iend = input->size != 0 ? src + input->size : src; + const char* ip = istart; + char* const dst = (char*)output->dst; + char* const ostart = output->pos != 0 ? dst + output->pos : dst; + char* const oend = output->size != 0 ? dst + output->size : dst; + char* op = ostart; + U32 someMoreWork = 1; + + DEBUGLOG(5, "ZSTD_decompressStream"); + RETURN_ERROR_IF( + input->pos > input->size, + srcSize_wrong, + "forbidden. in: pos: %u vs size: %u", + (U32)input->pos, (U32)input->size); + RETURN_ERROR_IF( + output->pos > output->size, + dstSize_tooSmall, + "forbidden. out: pos: %u vs size: %u", + (U32)output->pos, (U32)output->size); + DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); + FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), ""); + + while (someMoreWork) { + switch(zds->streamStage) + { + case zdss_init : + DEBUGLOG(5, "stage zdss_init => transparent reset "); + zds->streamStage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + zds->hostageByte = 0; + zds->expectedOutBuffer = *output; + ZSTD_FALLTHROUGH; + + case zdss_loadHeader : + DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); + { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format); +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + if (zds->refMultipleDDicts && zds->ddictSet) { + ZSTD_DCtx_selectFrameDDict(zds); + } +#endif + DEBUGLOG(5, "header size : %u", (U32)hSize); + if (ZSTD_isError(hSize)) { + return hSize; /* error */ + } + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + size_t const remainingInput = (size_t)(iend-ip); + assert(iend >= ip); + if (toLoad > remainingInput) { /* not enough input to load full header */ + if (remainingInput > 0) { + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput); + zds->lhSize += remainingInput; + } + input->pos = input->size; + return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + assert(ip != NULL); + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + break; + } } + + /* Check output buffer is large enough for ZSTD_odm_stable. */ + if (ZSTD_DCtx_get_outBufferMode(zds) == ZSTD_bm_stable + && zds->fParams.frameType != ZSTD_skippableFrame + && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) { + RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small"); + } + + /* Consume header (see ZSTDds_decodeFrameHeader) */ + DEBUGLOG(4, "Consume header"); + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); + + if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); + zds->stage = ZSTDds_skipFrame; + } else { + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), ""); + zds->expected = ZSTD_blockHeaderSize; + zds->stage = ZSTDds_decodeBlockHeader; + } + + /* control buffer memory usage */ + DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)", + (U32)(zds->fParams.windowSize >>10), + (U32)(zds->maxWindowSize >> 10) ); + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, + frameParameter_windowTooLarge, ""); + + /* Adapt buffer sizes to frame header instructions */ + { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); + size_t const neededOutBuffSize = ZSTD_DCtx_get_outBufferMode(zds) == ZSTD_bm_buffered + ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize) + : 0; + + ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); + + { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); + int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds); + + if (tooSmall || tooLarge) { + size_t const bufferSize = neededInBuffSize + neededOutBuffSize; + DEBUGLOG(4, "inBuff : from %u to %u", + (U32)zds->inBuffSize, (U32)neededInBuffSize); + DEBUGLOG(4, "outBuff : from %u to %u", + (U32)zds->outBuffSize, (U32)neededOutBuffSize); + if (ZSTD_DCtx_get_staticSize(zds)) { /* static DCtx */ + DEBUGLOG(4, "staticSize : %u", (U32)ZSTD_DCtx_get_staticSize(zds)); + assert(ZSTD_DCtx_get_staticSize(zds) >= sizeof(ZSTD_DCtx)); /* controlled at init */ + RETURN_ERROR_IF( + bufferSize > ZSTD_DCtx_get_staticSize(zds) - sizeof(ZSTD_DCtx), + memory_allocation, ""); + } else { + ZSTD_customFree(zds->inBuff, ZSTD_defaultCMem); + zds->inBuffSize = 0; + zds->outBuffSize = 0; + zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, ZSTD_defaultCMem); + RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, ""); + } + zds->inBuffSize = neededInBuffSize; + zds->outBuff = zds->inBuff + zds->inBuffSize; + zds->outBuffSize = neededOutBuffSize; + } } } + zds->streamStage = zdss_read; + ZSTD_FALLTHROUGH; + + case zdss_read: + DEBUGLOG(5, "stage zdss_read"); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)); + DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize); + if (neededInSize==0) { /* end of frame */ + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); + ip += neededInSize; + /* Function modifies the stage so we must break */ + break; + } } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->streamStage = zdss_load; + ZSTD_FALLTHROUGH; + + case zdss_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); + size_t const toLoad = neededInSize - zds->inPos; + int const isSkipFrame = ZSTD_isSkipFrame(zds); + size_t loadedSize; + /* At this point we shouldn't be decompressing a block that we can stream. */ + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip)); + if (isSkipFrame) { + loadedSize = MIN(toLoad, (size_t)(iend-ip)); + } else { + RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos, + corruption_detected, + "should never happen"); + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); + } + ip += loadedSize; + zds->inPos += loadedSize; + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + zds->inPos = 0; /* input is consumed */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), ""); + /* Function modifies the stage so we must break */ + break; + } + case zdss_flush: + { size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); + op += flushedSize; + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->streamStage = zdss_read; + if ( (zds->outBuffSize < zds->fParams.frameContentSize) + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", + (int)(zds->outBuffSize - zds->outStart), + (U32)zds->fParams.blockSizeMax); + zds->outStart = zds->outEnd = 0; + } + break; + } } + /* cannot complete flush */ + someMoreWork = 0; + break; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + } } + + /* result */ + input->pos = (size_t)(ip - (const char*)(input->src)); + output->pos = (size_t)(op - (char*)(output->dst)); + + /* Update the expected output buffer for ZSTD_obm_stable. */ + zds->expectedOutBuffer = *output; + + if ((ip==istart) && (op==ostart)) { /* no forward progress */ + zds->noForwardProgress ++; + if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { + RETURN_ERROR_IF(op==oend, dstSize_tooSmall, ""); + RETURN_ERROR_IF(ip==iend, srcSize_wrong, ""); + assert(0); + } + } else { + zds->noForwardProgress = 0; + } + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds); + if (!nextSrcSizeHint) { /* frame fully decoded */ + if (zds->outEnd == zds->outStart) { /* output fully flushed */ + if (zds->hostageByte) { + if (input->pos >= input->size) { + /* can't release hostage (not present) */ + zds->streamStage = zdss_read; + return 1; + } + input->pos++; /* release hostage */ + } /* zds->hostageByte */ + return 0; + } /* zds->outEnd == zds->outStart */ + if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ + input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ + zds->hostageByte=1; + } + return 1; + } /* nextSrcSizeHint==0 */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */ + assert(zds->inPos <= nextSrcSizeHint); + nextSrcSizeHint -= zds->inPos; /* part already loaded*/ + return nextSrcSizeHint; + } +} diff --git a/src/bled/zstd_decompress_block.c b/src/bled/zstd_decompress_block.c new file mode 100644 index 00000000000..0f44c1ed415 --- /dev/null +++ b/src/bled/zstd_decompress_block.c @@ -0,0 +1,1489 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_decompress_block : + * this module takes care of decompressing _compressed_ block */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "zstd_compiler.h" /* prefetch */ +#include "zstd_cpu.h" /* bmi2 */ +#include "zstd_mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_internal.h" +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" + +/*_******************************************************* +* Macros +**********************************************************/ + +/* These two optional macros force the use one way or another of the two + * ZSTD_decompressSequences implementations. You can't force in both directions + * at the same time. + */ +#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!" +#endif + + +/*_******************************************************* +* Memory operations +**********************************************************/ +static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } + + +/*-************************************************************* + * Block decoding + ***************************************************************/ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr) +{ + RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, ""); + + { U32 const cBlockHeader = MEM_readLE24(src); + U32 const cSize = cBlockHeader >> 3; + bpPtr->lastBlock = cBlockHeader & 1; + bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); + bpPtr->origSize = cSize; /* only useful for RLE */ + if (bpPtr->blockType == bt_rle) return 1; + RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); + return cSize; + } +} + + +/* Hidden declaration for fullbench */ +static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize); +/*! ZSTD_decodeLiteralsBlock() : + * @return : nb of bytes read from src (< srcSize ) + * note : symbol not declared but exposed for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ +{ + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); + RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); + + { const BYTE* const istart = (const BYTE*) src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + + switch(litEncType) + { + case set_repeat: + DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); + RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); + ZSTD_FALLTHROUGH; + + case set_compressed: + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3"); + { size_t lhSize, litSize, litCSize; + U32 singleStream=0; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + size_t hufSuccess; + switch(lhlCode) + { + case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); + break; + } + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); + +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + /* prefetch huffman table if cold */ + if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { + PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); + } +#endif + + if (litEncType==set_repeat) { + if (singleStream) { + hufSuccess = HUF_decompress1X_usingDTable_bmi2( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx)); + } else { + hufSuccess = HUF_decompress4X_usingDTable_bmi2( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx)); + } + } else { + if (singleStream) { +#if defined(HUF_FORCE_DECOMPRESS_X2) + hufSuccess = HUF_decompress1X_DCtx_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace)); +#else + hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx)); +#endif + } else { + hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx)); + } + } + + RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); + + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; + ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); + return litCSize + lhSize; + } + + case set_basic: + { size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + break; + } + + if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); + ZSTD_memcpy(dctx->litBuffer, istart+lhSize, litSize); + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); + return lhSize+litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart+lhSize; + dctx->litSize = litSize; + return lhSize+litSize; + } + + case set_rle: + { U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4"); + break; + } + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+1; + } + default: + RETURN_ERROR(corruption_detected, "impossible"); + } + } +} + +/* Default FSE distribution tables. + * These are pre-calculated FSE decoding tables using default distributions as defined in specification : + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions + * They were generated programmatically with following method : + * - start from default distributions, present in /lib/common/zstd_internal.h + * - generate tables normally, using ZSTD_buildFSETable() + * - printout the content of tables + * - pretify output, report below, test with fuzzer to ensure it's correct */ + +/* Default FSE distribution table for Literal Lengths */ +static const ZSTD_seqSymbol LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = { + { 1, 1, 1, LL_DEFAULTNORMLOG}, /* header : fastMode, tableLog */ + /* nextState, nbAddBits, nbBits, baseVal */ + { 0, 0, 4, 0}, { 16, 0, 4, 0}, + { 32, 0, 5, 1}, { 0, 0, 5, 3}, + { 0, 0, 5, 4}, { 0, 0, 5, 6}, + { 0, 0, 5, 7}, { 0, 0, 5, 9}, + { 0, 0, 5, 10}, { 0, 0, 5, 12}, + { 0, 0, 6, 14}, { 0, 1, 5, 16}, + { 0, 1, 5, 20}, { 0, 1, 5, 22}, + { 0, 2, 5, 28}, { 0, 3, 5, 32}, + { 0, 4, 5, 48}, { 32, 6, 5, 64}, + { 0, 7, 5, 128}, { 0, 8, 6, 256}, + { 0, 10, 6, 1024}, { 0, 12, 6, 4096}, + { 32, 0, 4, 0}, { 0, 0, 4, 1}, + { 0, 0, 5, 2}, { 32, 0, 5, 4}, + { 0, 0, 5, 5}, { 32, 0, 5, 7}, + { 0, 0, 5, 8}, { 32, 0, 5, 10}, + { 0, 0, 5, 11}, { 0, 0, 6, 13}, + { 32, 1, 5, 16}, { 0, 1, 5, 18}, + { 32, 1, 5, 22}, { 0, 2, 5, 24}, + { 32, 3, 5, 32}, { 0, 3, 5, 40}, + { 0, 6, 4, 64}, { 16, 6, 4, 64}, + { 32, 7, 5, 128}, { 0, 9, 6, 512}, + { 0, 11, 6, 2048}, { 48, 0, 4, 0}, + { 16, 0, 4, 1}, { 32, 0, 5, 2}, + { 32, 0, 5, 3}, { 32, 0, 5, 5}, + { 32, 0, 5, 6}, { 32, 0, 5, 8}, + { 32, 0, 5, 9}, { 32, 0, 5, 11}, + { 32, 0, 5, 12}, { 0, 0, 6, 15}, + { 32, 1, 5, 18}, { 32, 1, 5, 20}, + { 32, 2, 5, 24}, { 32, 2, 5, 28}, + { 32, 3, 5, 40}, { 32, 4, 5, 48}, + { 0, 16, 6,65536}, { 0, 15, 6,32768}, + { 0, 14, 6,16384}, { 0, 13, 6, 8192}, +}; /* LL_defaultDTable */ + +/* Default FSE distribution table for Offset Codes */ +static const ZSTD_seqSymbol OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = { + { 1, 1, 1, OF_DEFAULTNORMLOG}, /* header : fastMode, tableLog */ + /* nextState, nbAddBits, nbBits, baseVal */ + { 0, 0, 5, 0}, { 0, 6, 4, 61}, + { 0, 9, 5, 509}, { 0, 15, 5,32765}, + { 0, 21, 5,2097149}, { 0, 3, 5, 5}, + { 0, 7, 4, 125}, { 0, 12, 5, 4093}, + { 0, 18, 5,262141}, { 0, 23, 5,8388605}, + { 0, 5, 5, 29}, { 0, 8, 4, 253}, + { 0, 14, 5,16381}, { 0, 20, 5,1048573}, + { 0, 2, 5, 1}, { 16, 7, 4, 125}, + { 0, 11, 5, 2045}, { 0, 17, 5,131069}, + { 0, 22, 5,4194301}, { 0, 4, 5, 13}, + { 16, 8, 4, 253}, { 0, 13, 5, 8189}, + { 0, 19, 5,524285}, { 0, 1, 5, 1}, + { 16, 6, 4, 61}, { 0, 10, 5, 1021}, + { 0, 16, 5,65533}, { 0, 28, 5,268435453}, + { 0, 27, 5,134217725}, { 0, 26, 5,67108861}, + { 0, 25, 5,33554429}, { 0, 24, 5,16777213}, +}; /* OF_defaultDTable */ + + +/* Default FSE distribution table for Match Lengths */ +static const ZSTD_seqSymbol ML_defaultDTable[(1<<ML_DEFAULTNORMLOG)+1] = { + { 1, 1, 1, ML_DEFAULTNORMLOG}, /* header : fastMode, tableLog */ + /* nextState, nbAddBits, nbBits, baseVal */ + { 0, 0, 6, 3}, { 0, 0, 4, 4}, + { 32, 0, 5, 5}, { 0, 0, 5, 6}, + { 0, 0, 5, 8}, { 0, 0, 5, 9}, + { 0, 0, 5, 11}, { 0, 0, 6, 13}, + { 0, 0, 6, 16}, { 0, 0, 6, 19}, + { 0, 0, 6, 22}, { 0, 0, 6, 25}, + { 0, 0, 6, 28}, { 0, 0, 6, 31}, + { 0, 0, 6, 34}, { 0, 1, 6, 37}, + { 0, 1, 6, 41}, { 0, 2, 6, 47}, + { 0, 3, 6, 59}, { 0, 4, 6, 83}, + { 0, 7, 6, 131}, { 0, 9, 6, 515}, + { 16, 0, 4, 4}, { 0, 0, 4, 5}, + { 32, 0, 5, 6}, { 0, 0, 5, 7}, + { 32, 0, 5, 9}, { 0, 0, 5, 10}, + { 0, 0, 6, 12}, { 0, 0, 6, 15}, + { 0, 0, 6, 18}, { 0, 0, 6, 21}, + { 0, 0, 6, 24}, { 0, 0, 6, 27}, + { 0, 0, 6, 30}, { 0, 0, 6, 33}, + { 0, 1, 6, 35}, { 0, 1, 6, 39}, + { 0, 2, 6, 43}, { 0, 3, 6, 51}, + { 0, 4, 6, 67}, { 0, 5, 6, 99}, + { 0, 8, 6, 259}, { 32, 0, 4, 4}, + { 48, 0, 4, 4}, { 16, 0, 4, 5}, + { 32, 0, 5, 7}, { 32, 0, 5, 8}, + { 32, 0, 5, 10}, { 32, 0, 5, 11}, + { 0, 0, 6, 14}, { 0, 0, 6, 17}, + { 0, 0, 6, 20}, { 0, 0, 6, 23}, + { 0, 0, 6, 26}, { 0, 0, 6, 29}, + { 0, 0, 6, 32}, { 0, 16, 6,65539}, + { 0, 15, 6,32771}, { 0, 14, 6,16387}, + { 0, 13, 6, 8195}, { 0, 12, 6, 4099}, + { 0, 11, 6, 2051}, { 0, 10, 6, 1027}, +}; /* ML_defaultDTable */ + + +static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U32 nbAddBits) +{ + void* ptr = dt; + ZSTD_seqSymbol_header* const DTableH = (ZSTD_seqSymbol_header*)ptr; + ZSTD_seqSymbol* const cell = dt + 1; + + DTableH->tableLog = 0; + DTableH->fastMode = 0; + + cell->nbBits = 0; + cell->nextState = 0; + assert(nbAddBits < 255); + cell->nbAdditionalBits = (BYTE)nbAddBits; + cell->baseValue = baseValue; +} + + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * cannot fail if input is valid => + * all inputs are presumed validated at this stage */ +FORCE_INLINE_TEMPLATE +void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize) +{ + ZSTD_seqSymbol* const tableDecode = dt+1; + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + + U16* symbolNext = (U16*)wksp; + BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1); + U32 highThreshold = tableSize - 1; + + + /* Sanity Checks */ + assert(maxSymbolValue <= MaxSeq); + assert(tableLog <= MaxFSELog); + assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE); + (void)wkspSize; + /* Init, lay down lowprob symbols */ + { ZSTD_seqSymbol_header DTableH; + DTableH.tableLog = tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s<maxSV1; s++) { + if (normalizedCounter[s]==-1) { + tableDecode[highThreshold--].baseValue = s; + symbolNext[s] = 1; + } else { + if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; + assert(normalizedCounter[s]>=0); + symbolNext[s] = (U16)normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + assert(tableSize <= 512); + /* Specialized symbol spreading for the case when there are + * no low probability (-1 count) symbols. When compressing + * small blocks we avoid low probability symbols to hit this + * case, since header decoding speed matters more. + */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s<maxSV1; ++s, sv += add) { + int i; + int const n = normalizedCounter[s]; + MEM_write64(spread + pos, sv); + for (i = 8; i < n; i += 8) { + MEM_write64(spread + pos + i, sv); + } + pos += n; + } + } + /* Now we spread those positions across the table. + * The benefit of doing it in two stages is that we avoid the the + * variable size inner loop, which caused lots of branch misses. + * Now we can run through all the positions without any branch misses. + * We unroll the loop twice, since that is what emperically worked best. + */ + { + size_t position = 0; + size_t s; + size_t const unroll = 2; + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableDecode[uPosition].baseValue = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; + } + assert(position == 0); + } + } else { + U32 const tableMask = tableSize-1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s<maxSV1; s++) { + int i; + int const n = normalizedCounter[s]; + for (i=0; i<n; i++) { + tableDecode[position].baseValue = s; + position = (position + step) & tableMask; + while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { + U32 u; + for (u=0; u<tableSize; u++) { + U32 const symbol = tableDecode[u].baseValue; + U32 const nextState = symbolNext[symbol]++; + tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) ); + tableDecode[u].nextState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); + assert(nbAdditionalBits[symbol] < 255); + tableDecode[u].nbAdditionalBits = (BYTE)nbAdditionalBits[symbol]; + tableDecode[u].baseValue = baseValue[symbol]; + } + } +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static void ZSTD_buildFSETable_body_default(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize) +{ + ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue, + baseValue, nbAdditionalBits, tableLog, wksp, wkspSize); +} + +#if DYNAMIC_BMI2 +TARGET_ATTRIBUTE("bmi2") static void ZSTD_buildFSETable_body_bmi2(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize) +{ + ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue, + baseValue, nbAdditionalBits, tableLog, wksp, wkspSize); +} +#endif + +void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + ZSTD_buildFSETable_body_bmi2(dt, normalizedCounter, maxSymbolValue, + baseValue, nbAdditionalBits, tableLog, wksp, wkspSize); + return; + } +#endif + (void)bmi2; + ZSTD_buildFSETable_body_default(dt, normalizedCounter, maxSymbolValue, + baseValue, nbAdditionalBits, tableLog, wksp, wkspSize); +} + + +/*! ZSTD_buildSeqTable() : + * @return : nb bytes read from src, + * or an error code if it fails */ +static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymbol** DTablePtr, + symbolEncodingType_e type, unsigned max, U32 maxLog, + const void* src, size_t srcSize, + const U32* baseValue, const U32* nbAdditionalBits, + const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable, + int ddictIsCold, int nbSeq, U32* wksp, size_t wkspSize, + int bmi2) +{ + switch(type) + { + case set_rle : + RETURN_ERROR_IF(!srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF((*(const BYTE*)src) > max, corruption_detected, ""); + { U32 const symbol = *(const BYTE*)src; + U32 const baseline = baseValue[symbol]; + U32 const nbBits = nbAdditionalBits[symbol]; + ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); + } + *DTablePtr = DTableSpace; + return 1; + case set_basic : + *DTablePtr = defaultTable; + return 0; + case set_repeat: + RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, ""); + /* prefetch FSE table if used */ + if (ddictIsCold && (nbSeq > 24 /* heuristic */)) { + const void* const pStart = *DTablePtr; + size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog)); + PREFETCH_AREA(pStart, pSize); + } + return 0; + case set_compressed : + { unsigned tableLog; + S16 norm[MaxSeq+1]; + size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); + RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, ""); + RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, ""); + ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2); + *DTablePtr = DTableSpace; + return headerSize; + } + default : + assert(0); + RETURN_ERROR(GENERIC, "impossible"); + } +} + +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + int nbSeq; + DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); + + /* check */ + RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, ""); + + /* SeqHead */ + nbSeq = *ip++; + if (!nbSeq) { + *nbSeqPtr=0; + RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, ""); + return 1; + } + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) { + RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); + nbSeq = MEM_readLE16(ip) + LONGNBSEQ; + ip+=2; + } else { + RETURN_ERROR_IF(ip >= iend, srcSize_wrong, ""); + nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } + } + *nbSeqPtr = nbSeq; + + /* FSE table descriptors */ + RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ + { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); + symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); + symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + int ddictIsCold = 0; + ip++; +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + ddictIsCold = dctx->ddictIsCold; +#endif + + /* Build DTables */ + { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, + LLtype, MaxLL, LLFSELog, + ip, iend-ip, + LL_base, LL_bits, + LL_defaultDTable, dctx->fseEntropy, + ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += llhSize; + } + + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, + OFtype, MaxOff, OffFSELog, + ip, iend-ip, + OF_base, OF_bits, + OF_defaultDTable, dctx->fseEntropy, + ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += ofhSize; + } + + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, + MLtype, MaxML, MLFSELog, + ip, iend-ip, + ML_base, ML_bits, + ML_defaultDTable, dctx->fseEntropy, + ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += mlhSize; + } + } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; +} seq_t; + +typedef struct { + size_t state; + const ZSTD_seqSymbol* table; +} ZSTD_fseState; + +typedef struct { + BIT_DStream_t DStream; + ZSTD_fseState stateLL; + ZSTD_fseState stateOffb; + ZSTD_fseState stateML; + size_t prevOffset[ZSTD_REP_NUM]; +} seqState_t; + +/*! ZSTD_overlapCopy8() : + * Copies 8 bytes from ip to op and updates op and ip where ip <= op. + * If the offset is < 8 then the offset is spread to at least 8 bytes. + * + * Precondition: *ip <= *op + * Postcondition: *op - *op >= 8 + */ +HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { + assert(*ip <= *op); + if (offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[offset]; + (*op)[0] = (*ip)[0]; + (*op)[1] = (*ip)[1]; + (*op)[2] = (*ip)[2]; + (*op)[3] = (*ip)[3]; + *ip += dec32table[offset]; + ZSTD_copy4(*op+4, *ip); + *ip -= sub2; + } else { + ZSTD_copy8(*op, *ip); + } + *ip += 8; + *op += 8; + assert(*op - *ip >= 8); +} + +/*! ZSTD_safecopy() : + * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer + * and write up to 16 bytes past oend_w (op >= oend_w is allowed). + * This function is only called in the uncommon case where the sequence is near the end of the block. It + * should be fast for a single long sequence, but can be slow for several short sequences. + * + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. + * The src buffer must be before the dst buffer. + */ +static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) || + (ovtype == ZSTD_overlap_src_before_dst && diff >= 0)); + + if (length < 8) { + /* Handle short lengths. */ + while (op < oend) *op++ = *ip++; + return; + } + if (ovtype == ZSTD_overlap_src_before_dst) { + /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ + assert(length >= 8); + ZSTD_overlapCopy8(&op, &ip, diff); + assert(op - ip >= 8); + assert(op <= oend); + } + + if (oend <= oend_w) { + /* No risk of overwrite. */ + ZSTD_wildcopy(op, ip, length, ovtype); + return; + } + if (op <= oend_w) { + /* Wildcopy until we get close to the end. */ + assert(oend > oend_w); + ZSTD_wildcopy(op, ip, oend_w - op, ovtype); + ip += oend_w - op; + op = oend_w; + } + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_execSequenceEnd(): + * This version handles cases that are near the end of the output buffer. It requires + * more careful checks to make sure there is no overflow. By separating out these hard + * and unlikely cases, we can speed up the common cases. + * + * NOTE: This function needs to be fast for a single long sequence, but doesn't need + * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). + */ +FORCE_NOINLINE +size_t ZSTD_execSequenceEnd(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart-match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +HINT_INLINE +size_t ZSTD_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + +static void +ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) +{ + const void* ptr = dt; + const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits", + (U32)DStatePtr->state, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +FORCE_INLINE_TEMPLATE void +ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits) +{ + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = nextState + lowBits; +} + +/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum + * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1) + * bits before reloading. This value is the maximum number of bytes we read + * after reloading when we are decoding long offsets. + */ +#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ + (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \ + ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \ + : 0) + +typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; + +FORCE_INLINE_TEMPLATE seq_t +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) +{ + seq_t seq; + const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; + const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; + const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; + seq.matchLength = mlDInfo->baseValue; + seq.litLength = llDInfo->baseValue; + { U32 const ofBase = ofDInfo->baseValue; + BYTE const llBits = llDInfo->nbAdditionalBits; + BYTE const mlBits = mlDInfo->nbAdditionalBits; + BYTE const ofBits = ofDInfo->nbAdditionalBits; + BYTE const totalBits = llBits+mlBits+ofBits; + + U16 const llNext = llDInfo->nextState; + U16 const mlNext = mlDInfo->nextState; + U16 const ofNext = ofDInfo->nextState; + U32 const llnbBits = llDInfo->nbBits; + U32 const mlnbBits = mlDInfo->nbBits; + U32 const ofnbBits = ofDInfo->nbBits; + /* + * As gcc has better branch and block analyzers, sometimes it is only + * valuable to mark likelyness for clang, it gives around 3-4% of + * performance. + */ + + /* sequence */ + { size_t offset; + #if defined(__clang__) + if (LIKELY(ofBits > 1)) { + #else + if (ofBits > 1) { + #endif + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + assert(ofBits <= MaxOff); + if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { + U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed); + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + BIT_reloadDStream(&seqState->DStream); + if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); + assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */ + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } else { + U32 const ll0 = (llDInfo->baseValue == 0); + if (LIKELY((ofBits == 0))) { + offset = seqState->prevOffset[ll0]; + seqState->prevOffset[1] = seqState->prevOffset[!ll0]; + seqState->prevOffset[0] = offset; + } else { + offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); + { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } } } + seq.offset = offset; + } + + #if defined(__clang__) + if (UNLIKELY(mlBits > 0)) + #else + if (mlBits > 0) + #endif + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); + + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + #if defined(__clang__) + if (UNLIKELY(llBits > 0)) + #else + if (llBits > 0) + #endif + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); + + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + } + + return seq; +} + + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body"); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + +#if defined(__GNUC__) && defined(__x86_64__) + /* Align the decompression loop to 32 + 16 bytes. + * + * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression + * speed swings based on the alignment of the decompression loop. This + * performance swing is caused by parts of the decompression loop falling + * out of the DSB. The entire decompression loop should fit in the DSB, + * when it can't we get much worse performance. You can measure if you've + * hit the good case or the bad case with this perf command for some + * compressed file test.zst: + * + * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ + * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst + * + * If you see most cycles served out of the MITE you've hit the bad case. + * If you see most cycles served out of the DSB you've hit the good case. + * If it is pretty even then you may be in an okay case. + * + * This issue has been reproduced on the following CPUs: + * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 + * Use Instruments->Counters to get DSB/MITE cycles. + * I never got performance swings, but I was able to + * go from the good case of mostly DSB to half of the + * cycles served from MITE. + * - Coffeelake: Intel i9-9900k + * - Coffeelake: Intel i7-9700k + * + * I haven't been able to reproduce the instability or DSB misses on any + * of the following CPUS: + * - Haswell + * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH + * - Skylake + * + * If you are seeing performance stability this script can help test. + * It tests on 4 commits in zstd where I saw performance change. + * + * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 + */ + __asm__(".p2align 6"); + __asm__("nop"); + __asm__(".p2align 5"); + __asm__("nop"); +# if __GNUC__ >= 9 && __GNUC__ < 11 + /* better for gcc-9 and gcc-10, worse for clang and gcc-8, gcc-11 */ + __asm__(".p2align 3"); +# else + __asm__(".p2align 4"); +# endif +#endif + for ( ; ; ) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (UNLIKELY(!--nbSeq)) + break; + BIT_reloadDStream(&(seqState.DStream)); + } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); + RETURN_ERROR_IF(nbSeq, corruption_detected, ""); + RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +static size_t +ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + +FORCE_INLINE_TEMPLATE size_t +ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence, + const BYTE* const prefixStart, const BYTE* const dictEnd) +{ + prefetchPos += sequence.litLength; + { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart; + const BYTE* const match = matchBase + prefetchPos - sequence.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : memory address is only used for prefetching, not for dereferencing */ + PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ + } + return prefetchPos + sequence.matchLength; +} + +/* This decoding function employs prefetching + * to reduce latency impact of cache misses. + * It's generally employed when block contains a significant portion of long-distance matches + * or when coupled with a "cold" dictionary */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_decompressSequencesLong_body( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { +#define STORED_SEQS 8 +#define STORED_SEQS_MASK (STORED_SEQS-1) +#define ADVANCED_SEQS STORED_SEQS + seq_t sequences[STORED_SEQS]; + int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); + seqState_t seqState; + int seqNb; + size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */ + + dctx->fseEntropy = 1; + { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } + assert(dst != NULL); + assert(iend >= ip); + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + /* prepare in advance */ + for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNb<seqAdvance); seqNb++) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb] = sequence; + } + RETURN_ERROR_IF(seqNb<seqAdvance, corruption_detected, ""); + + /* decode and decompress */ + for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb<nbSeq) ; seqNb++) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd); + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } + RETURN_ERROR_IF(seqNb<nbSeq, corruption_detected, ""); + + /* finish queue */ + seqNb -= seqAdvance; + for ( ; seqNb<nbSeq ; seqNb++) { + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[seqNb&STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd); + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + + /* save reps for next block */ + { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +static size_t +ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + + +#if DYNAMIC_BMI2 + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static TARGET_ATTRIBUTE("bmi2") size_t +DONT_VECTORIZE +ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +static TARGET_ATTRIBUTE("bmi2") size_t +ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + +#endif /* DYNAMIC_BMI2 */ + +typedef size_t (*ZSTD_decompressSequences_t)( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame); + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static size_t +ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequences"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +/* ZSTD_decompressSequencesLong() : + * decompression function triggered when a minimum share of offsets is considered "long", + * aka out of cache. + * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance". + * This function will try to mitigate main memory latency through the use of prefetching */ +static size_t +ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesLong"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +/* ZSTD_getLongOffsetsShare() : + * condition : offTable must be valid + * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) + * compared to maximum possible of (1<<OffFSELog) */ +static unsigned +ZSTD_getLongOffsetsShare(const ZSTD_seqSymbol* offTable) +{ + const void* ptr = offTable; + U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog; + const ZSTD_seqSymbol* table = offTable + 1; + U32 const max = 1 << tableLog; + U32 u, total = 0; + DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog); + + assert(max <= (1 << OffFSELog)); /* max not too large */ + for (u=0; u<max; u++) { + if (table[u].nbAdditionalBits > 22) total += 1; + } + + assert(tableLog <= OffFSELog); + total <<= (OffFSELog - tableLog); /* scale to OffFSELog */ + + return total; +} +#endif + +size_t +ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN. + * We don't expect that to be the case in 64-bit mode. + * In block mode, window size is not known, so we have to be conservative. + * (note: but it could be evaluated from current-lowLimit) + */ + ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)))); + DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); + + RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); + + /* Decode literals section */ + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); + if (ZSTD_isError(litCSize)) return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + + /* Build Decoding Tables */ + { + /* These macros control at build-time which decompressor implementation + * we use. If neither is defined, we do some inspection and dispatch at + * runtime. + */ +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + int usePrefetchDecoder = 0; +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + usePrefetchDecoder = dctx->ddictIsCold; +#endif +#endif + int nbSeq; + size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + srcSize -= seqHSize; + + RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if ( !usePrefetchDecoder + && (!frame || (dctx->fParams.windowSize > (1<<24))) + && (nbSeq>ADVANCED_SEQS) ) { /* could probably use a larger nbSeq limit */ + U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (shareLongOffsets >= minShare); + } +#endif +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + dctx->ddictIsCold = 0; +#endif + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if (usePrefetchDecoder) +#endif +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); +#endif + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + /* else */ + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); +#endif + } +} + + +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) +{ + if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dst; + dctx->previousDstEnd = dst; + } +} + + +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + ZSTD_checkContinuity(dctx, dst, dstCapacity); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} diff --git a/src/bled/zstd_decompress_block.h b/src/bled/zstd_decompress_block.h new file mode 100644 index 00000000000..47653414508 --- /dev/null +++ b/src/bled/zstd_decompress_block.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DEC_BLOCK_H +#define ZSTD_DEC_BLOCK_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "zstd_deps.h" /* size_t */ +#include "zstd.h" /* DCtx, and some public functions */ +#include "zstd_internal.h" /* blockProperties_t, and some public functions */ +#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */ + + +/* === Prototypes === */ + +/* note: prototypes already published within `zstd.h` : + * ZSTD_decompressBlock() + */ + +/* note: prototypes already published within `zstd_internal.h` : + * ZSTD_getcBlockSize() + * ZSTD_decodeSeqHeaders() + */ + + +/* ZSTD_decompressBlock_internal() : + * decompress block, starting at `src`, + * into destination buffer `dst`. + * @return : decompressed block size, + * or an error code (which can be tested using ZSTD_isError()) + */ +size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame); + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * this function must be called with valid parameters only + * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.) + * in which case it cannot fail. + * The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is + * defined in zstd_decompress_internal.h. + * Internal use only. + */ +void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize, + int bmi2); + + +#endif /* ZSTD_DEC_BLOCK_H */ diff --git a/src/bled/zstd_decompress_internal.h b/src/bled/zstd_decompress_internal.h new file mode 100644 index 00000000000..19841485580 --- /dev/null +++ b/src/bled/zstd_decompress_internal.h @@ -0,0 +1,205 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* zstd_decompress_internal: + * objects and definitions shared within lib/decompress modules */ + + #ifndef ZSTD_DECOMPRESS_INTERNAL_H + #define ZSTD_DECOMPRESS_INTERNAL_H + + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "zstd_mem.h" /* BYTE, U16, U32 */ +#include "zstd_internal.h" /* ZSTD_seqSymbol */ + + + +/*-******************************************************* + * Constants + *********************************************************/ +static UNUSED_ATTR const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + +static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + +static UNUSED_ATTR const U32 OF_bits[MaxOff+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31 }; + +static UNUSED_ATTR const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + +/*-******************************************************* + * Decompression types + *********************************************************/ + typedef struct { + U32 fastMode; + U32 tableLog; + } ZSTD_seqSymbol_header; + + typedef struct { + U16 nextState; + BYTE nbAdditionalBits; + BYTE nbBits; + U32 baseValue; + } ZSTD_seqSymbol; + + #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) + +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) + +typedef struct { + ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ + ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ + ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ + U32 rep[ZSTD_REP_NUM]; + U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; +} ZSTD_entropyDTables_t; + +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; + +typedef enum { zdss_init=0, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +typedef enum { + ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */ + ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */ + ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ +} ZSTD_dictUses_e; + +/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */ +typedef struct { + const ZSTD_DDict** ddictPtrTable; + size_t ddictPtrTableSize; + size_t ddictPtrCount; +} ZSTD_DDictHashSet; + +struct ZSTD_DCtx_s +{ + const ZSTD_seqSymbol* LLTptr; + const ZSTD_seqSymbol* MLTptr; + const ZSTD_seqSymbol* OFTptr; + const HUF_DTable* HUFptr; + ZSTD_entropyDTables_t entropy; + U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ + const void* previousDstEnd; /* detect continuity */ + const void* prefixStart; /* start of current segment */ + const void* virtualStart; /* virtual start of previous segment if it was just before current one */ + const void* dictEnd; /* end of previous segment */ + size_t expected; + ZSTD_frameHeader fParams; + U64 processedCSize; + U64 decodedSize; + blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + ZSTD_format_e format; + ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */ + U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */ + const BYTE* litPtr; + size_t litSize; + size_t rleSize; +#if DYNAMIC_BMI2 != 0 + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ +#endif + +#if ZSTD_DECOMPRESS_DICTIONARY != 0 + /* dictionary */ + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ + U32 dictID; + int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ + ZSTD_dictUses_e dictUses; + ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ + ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ +#endif + + /* streaming */ + ZSTD_dStreamStage streamStage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t lhSize; + U32 hostageByte; + int noForwardProgress; + ZSTD_outBuffer expectedOutBuffer; + + /* workspace */ + BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + + size_t oversizedDuration; + + + /* Tracing */ +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + +MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { +#if DYNAMIC_BMI2 != 0 + return dctx->bmi2; +#else + (void)dctx; + return 0; +#endif +} +#define ZSTD_DCtx_get_staticSize(dctx) (0) +#define ZSTD_DCtx_get_outBufferMode(dctx) (ZSTD_bm_buffered) +/*-******************************************************* + * Shared internal functions + *********************************************************/ + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */ +size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize); + +/*! ZSTD_checkContinuity() : + * check if next `dst` follows previous position, where decompression ended. + * If yes, do nothing (continue on current segment). + * If not, classify previous segment as "external dictionary", and start a new segment. + * This function cannot fail. */ +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize); + + +#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ diff --git a/src/bled/zstd_deps.h b/src/bled/zstd_deps.h new file mode 100644 index 00000000000..8a62920edf0 --- /dev/null +++ b/src/bled/zstd_deps.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This file provides common libc dependencies that zstd requires. + * The purpose is to allow replacing this file with a custom implementation + * to compile zstd without libc support. + */ + +/* Need: + * NULL + * INT_MAX + * UINT_MAX + * ZSTD_memcpy() + * ZSTD_memset() + * ZSTD_memmove() + */ +#ifndef ZSTD_DEPS_COMMON +#define ZSTD_DEPS_COMMON + +#include <assert.h> +#include <limits.h> +#include <stddef.h> +#include <string.h> +#include <crtdefs.h> +#include "zstd_config.h" + +#if defined(__GNUC__) && __GNUC__ >= 4 +# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l)) +#else +# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) memset((p),(v),(l)) +#endif + +#endif /* ZSTD_DEPS_COMMON */ + +/* Need: + * ZSTD_malloc() + * ZSTD_free() + * ZSTD_calloc() + */ +#ifdef ZSTD_DEPS_NEED_MALLOC +#ifndef ZSTD_DEPS_MALLOC +#define ZSTD_DEPS_MALLOC + +#include <stdlib.h> + +#define ZSTD_malloc(s) malloc(s) +#define ZSTD_calloc(n,s) calloc((n), (s)) +#define ZSTD_free(p) free((p)) + +#endif /* ZSTD_DEPS_MALLOC */ +#endif /* ZSTD_DEPS_NEED_MALLOC */ + +/* + * Provides 64-bit math support. + * Need: + * U64 ZSTD_div64(U64 dividend, U32 divisor) + */ +#ifdef ZSTD_DEPS_NEED_MATH64 +#ifndef ZSTD_DEPS_MATH64 +#define ZSTD_DEPS_MATH64 + +#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor)) + +#endif /* ZSTD_DEPS_MATH64 */ +#endif /* ZSTD_DEPS_NEED_MATH64 */ + +/* Need: + * assert() + */ +#ifdef ZSTD_DEPS_NEED_ASSERT +#ifndef ZSTD_DEPS_ASSERT +#define ZSTD_DEPS_ASSERT + +#include <assert.h> + +#endif /* ZSTD_DEPS_ASSERT */ +#endif /* ZSTD_DEPS_NEED_ASSERT */ + +/* Need: + * ZSTD_DEBUG_PRINT() + */ +#ifdef ZSTD_DEPS_NEED_IO +#ifndef ZSTD_DEPS_IO +#define ZSTD_DEPS_IO + +#include <stdio.h> +#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__) + +#endif /* ZSTD_DEPS_IO */ +#endif /* ZSTD_DEPS_NEED_IO */ + +/* Only requested when <stdint.h> is known to be present. + * Need: + * intptr_t + */ +#ifdef ZSTD_DEPS_NEED_STDINT +#ifndef ZSTD_DEPS_STDINT +#define ZSTD_DEPS_STDINT + +#include <stdint.h> + +#endif /* ZSTD_DEPS_STDINT */ +#endif /* ZSTD_DEPS_NEED_STDINT */ + +#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1]) +#include "libbb.h" +#define RAWLOG(l, ...) { if (l <= ZSTD_DEBUGLEVEL) bb_printf(__VA_ARGS__); } +#define DEBUGLOG(l, ...) { if (l <= ZSTD_DEBUGLEVEL) bb_printf(__VA_ARGS__); } + +#if defined(__GNUC__) +# define MEM_STATIC static __inline __attribute__((unused)) +#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + diff --git a/src/bled/zstd_entropy_common.c b/src/bled/zstd_entropy_common.c new file mode 100644 index 00000000000..5103f0f0f28 --- /dev/null +++ b/src/bled/zstd_entropy_common.c @@ -0,0 +1,362 @@ +/* ****************************************************************** + * Common functions of New Generation Entropy library + * Copyright (c) Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************* +* Dependencies +***************************************/ +#include "zstd_mem.h" +#include "zstd_error_private.h" /* ERR_*, ERROR */ +#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ +#include "huf.h" + + +/*=== Version ===*/ +unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } + + +/*=== Error Management ===*/ +#define FSE_isError ERR_isError +const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } + +#define HUF_isError ERR_isError +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +static U32 FSE_ctz(U32 val) +{ + assert(val != 0); + { +# if defined(_MSC_VER) /* Visual */ + unsigned long r=0; + return _BitScanForward(&r, val) ? (unsigned)r : 0; +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ + return __builtin_ctz(val); +# elif defined(__ICCARM__) /* IAR Intrinsic */ + return __CTZ(val); +# else /* Software version */ + U32 count = 0; + while ((val & 1) == 0) { + val >>= 1; + ++count; + } + return count; +# endif + } +} + +FORCE_INLINE_TEMPLATE +size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + unsigned const maxSV1 = *maxSVPtr + 1; + int previous0 = 0; + + if (hbSize < 8) { + /* This function only works when hbSize >= 8 */ + char buffer[8] = {0}; + ZSTD_memcpy(buffer, headerBuffer, hbSize); + { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr, + buffer, sizeof(buffer)); + if (FSE_isError(countSize)) return countSize; + if (countSize > hbSize) return ERROR(corruption_detected); + return countSize; + } } + assert(hbSize >= 8); + + /* init */ + ZSTD_memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */ + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<<nbBits)+1; + threshold = 1<<nbBits; + nbBits++; + + for (;;) { + if (previous0) { + /* Count the number of repeats. Each time the + * 2-bit repeat code is 0b11 there is another + * repeat. + * Avoid UB by setting the high bit to 1. + */ + int repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + while (repeats >= 12) { + charnum += 3 * 12; + if (LIKELY(ip <= iend-7)) { + ip += 3; + } else { + bitCount -= (int)(8 * (iend - 7 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + } + charnum += 3 * repeats; + bitStream >>= 2 * repeats; + bitCount += 2 * repeats; + + /* Add the final repeat which isn't 0b11. */ + assert((bitStream & 3) < 3); + charnum += bitStream & 3; + bitCount += 2; + + /* This is an error, but break and return an error + * at the end, because returning out of a loop makes + * it harder for the compiler to optimize. + */ + if (charnum >= maxSV1) break; + + /* We don't need to set the normalized count to 0 + * because we already memset the whole buffer to 0. + */ + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + assert((bitCount >> 3) <= 3); /* For first condition to work */ + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } + { + int const max = (2*threshold-1) - remaining; + int count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = bitStream & (threshold-1); + bitCount += nbBits-1; + } else { + count = bitStream & (2*threshold-1); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + /* When it matters (small blocks), this is a + * predictable branch, because we don't use -1. + */ + if (count >= 0) { + remaining -= count; + } else { + assert(count == -1); + remaining += count; + } + normalizedCounter[charnum++] = (short)count; + previous0 = !count; + + assert(threshold > 1); + if (remaining < threshold) { + /* This branch can be folded into the + * threshold update condition because we + * know that threshold > 1. + */ + if (remaining <= 1) break; + nbBits = BIT_highbit32(remaining) + 1; + threshold = 1 << (nbBits - 1); + } + if (charnum >= maxSV1) break; + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } } + if (remaining != 1) return ERROR(corruption_detected); + /* Only possible when there are too many zeros. */ + if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall); + if (bitCount > 32) return ERROR(corruption_detected); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + return ip-istart; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_readNCount_body_default( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +#if DYNAMIC_BMI2 +TARGET_ATTRIBUTE("bmi2") static size_t FSE_readNCount_body_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} +#endif + +size_t FSE_readNCount_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); + } +#endif + (void)bmi2; + return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +size_t FSE_readNCount( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0); +} + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableX?() . +*/ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int bmi2) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize; + size_t oSize; + + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; + /* ZSTD_memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n<oSize; n+=2) { + huffWeight[n] = ip[n/2] >> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + /* max (hwSize-1) values decoded, as last one is implied */ + oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2); + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n<oSize; n++) { + if (huffWeight[n] > HUF_TABLELOG_MAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + if (weightTotal == 0) return ERROR(corruption_detected); + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << BIT_highbit32(rest); + U32 const lastWeight = BIT_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +static TARGET_ATTRIBUTE("bmi2") size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1); +} +#endif + +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); + } +#endif + (void)bmi2; + return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); +} diff --git a/src/bled/zstd_error_private.c b/src/bled/zstd_error_private.c new file mode 100644 index 00000000000..531d1ee7b49 --- /dev/null +++ b/src/bled/zstd_error_private.c @@ -0,0 +1,56 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "zstd_error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ +#ifdef ZSTD_STRIP_ERROR_STRINGS + (void)code; + return "Error strings stripped"; +#else + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; + case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size is incorrect"; + case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; + /* following error codes are not stable and may be removed or changed in a future version */ + case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; + case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; + case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; + case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; + case PREFIX(maxCode): + default: return notErrorCode; + } +#endif +} diff --git a/src/bled/zstd_error_private.h b/src/bled/zstd_error_private.h new file mode 100644 index 00000000000..fa75e32a01c --- /dev/null +++ b/src/bled/zstd_error_private.h @@ -0,0 +1,157 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + + + +/* **************************************** +* Dependencies +******************************************/ +#include "zstd_errors.h" /* enum list */ +#include "zstd_compiler.h" +#include "zstd_deps.h" /* size_t */ + + +/* **************************************** +* Compiler-specific +******************************************/ +#if defined(__GNUC__) +# define ERR_STATIC static __attribute__((unused)) +#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define ERR_STATIC static inline +#elif defined(_MSC_VER) +# define ERR_STATIC static __inline +#else +# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Customization (error_public.h) +******************************************/ +typedef ZSTD_ErrorCode ERR_enum; +#define PREFIX(name) ZSTD_error_##name + + +/*-**************************************** +* Error codes handling +******************************************/ +#undef ERROR /* already defined on Visual Studio */ +#define ERROR(name) ZSTD_ERROR(name) +#define ZSTD_ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } + +/* check and forward error code */ +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } + + +/*-**************************************** +* Error Strings +******************************************/ + +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ +#ifdef ZSTD_STRIP_ERROR_STRINGS + (void)code; + return "Error strings stripped"; +#else + return ERR_getErrorString(ERR_getErrorCode(code)); +#endif +} + +/** + * Ignore: this is an internal helper. + * + * This is a helper function to help force C99-correctness during compilation. + * Under strict compilation modes, variadic macro arguments can't be empty. + * However, variadic function arguments can be. Using a function therefore lets + * us statically check that at least one (string) argument was passed, + * independent of the compilation flags. + */ +static INLINE_KEYWORD UNUSED_ATTR +void _force_has_format_string(const char *format, ...) { + (void)format; +} + +/** + * Ignore: this is an internal helper. + * + * We want to force this function invocation to be syntactically correct, but + * we don't want to force runtime evaluation of its arguments. + */ +#define _FORCE_HAS_FORMAT_STRING(...) \ + if (0) { \ + _force_has_format_string(__VA_ARGS__); \ + } + +#define ERR_QUOTE(str) #str + +/** + * Return the specified error if the condition evaluates to true. + * + * In debug modes, prints additional information. + * In order to do that (particularly, printing the conditional that failed), + * this can't just wrap RETURN_ERROR(). + */ +#define RETURN_ERROR_IF(cond, err, ...) \ + if (cond) { \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } + +/** + * Unconditionally return the specified error. + * + * In debug modes, prints additional information. + */ +#define RETURN_ERROR(err, ...) \ + do { \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } while(0); + +/** + * If the provided expression evaluates to an error code, returns that error code. + * + * In debug modes, prints additional information. + */ +#define FORWARD_IF_ERROR(err, ...) \ + do { \ + size_t const err_code = (err); \ + if (ERR_isError(err_code)) { \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ + __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return err_code; \ + } \ + } while(0); + + +#endif /* ERROR_H_MODULE */ diff --git a/src/bled/zstd_errors.h b/src/bled/zstd_errors.h new file mode 100644 index 00000000000..576d46dfb65 --- /dev/null +++ b/src/bled/zstd_errors.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + + +/*===== dependency =====*/ +#include "zstd_deps.h" + + +/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ +#define ZSTDERRORLIB_VISIBILITY MEM_STATIC +#define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY + +/*-********************************************* + * Error codes list + *-********************************************* + * Error codes _values_ are pinned down since v1.3.1 only. + * Therefore, don't rely on values if you may link to any version < v1.3.1. + * + * Only values < 100 are considered stable. + * + * note 1 : this API shall be used with static linking only. + * dynamic linking is not yet officially supported. + * note 2 : Prefer relying on the enum than on its value whenever possible + * This is the only supported way to use the error list < v1.3.1 + * note 3 : ZSTD_isError() is always correct, whatever the library version. + **********************************************/ +typedef enum { + ZSTD_error_no_error = 0, + ZSTD_error_GENERIC = 1, + ZSTD_error_prefix_unknown = 10, + ZSTD_error_version_unsupported = 12, + ZSTD_error_frameParameter_unsupported = 14, + ZSTD_error_frameParameter_windowTooLarge = 16, + ZSTD_error_corruption_detected = 20, + ZSTD_error_checksum_wrong = 22, + ZSTD_error_dictionary_corrupted = 30, + ZSTD_error_dictionary_wrong = 32, + ZSTD_error_dictionaryCreation_failed = 34, + ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_outOfBound = 42, + ZSTD_error_tableLog_tooLarge = 44, + ZSTD_error_maxSymbolValue_tooLarge = 46, + ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_stage_wrong = 60, + ZSTD_error_init_missing = 62, + ZSTD_error_memory_allocation = 64, + ZSTD_error_workSpace_tooSmall= 66, + ZSTD_error_dstSize_tooSmall = 70, + ZSTD_error_srcSize_wrong = 72, + ZSTD_error_dstBuffer_null = 74, + /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ + ZSTD_error_frameIndex_tooLarge = 100, + ZSTD_error_seekableIO = 102, + ZSTD_error_dstBuffer_wrong = 104, + ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ +} ZSTD_ErrorCode; + +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare with enum list published above */ +ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ + + + +#endif /* ZSTD_ERRORS_H_398273423 */ diff --git a/src/bled/zstd_internal.h b/src/bled/zstd_internal.h new file mode 100644 index 00000000000..549a259e3ea --- /dev/null +++ b/src/bled/zstd_internal.h @@ -0,0 +1,449 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CCOMMON_H_MODULE +#define ZSTD_CCOMMON_H_MODULE + +/* this module contains definitions which must be identical + * across compression, decompression and dictBuilder. + * It also contains a few functions useful to at least 2 of them + * and which benefit from being inlined */ + +/*-************************************* +* Dependencies +***************************************/ +#include "zstd_compiler.h" +#include "zstd_mem.h" +#include "zstd_error_private.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "xxhash.h" /* XXH_reset, update, digest */ +#define ZSTD_TRACE 0 + + +/* ---- static assert (debug) --- */ +#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + +/*-************************************* +* shared macros +***************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) + +#define ZSTD_REP_NUM 3 /* number of repcodes */ +#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) +static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ + +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define ZSTD_FRAMECHECKSUMSIZE 4 + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ + +#define HufLog 12 +typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 + +#define Litbits 8 +#define MaxLit ((1<<Litbits) - 1) +#define MaxML 52 +#define MaxLL 35 +#define DefaultMaxOff 28 +#define MaxOff 31 +#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */ +#define MLFSELog 9 +#define LLFSELog 9 +#define OffFSELog 8 +#define MaxFSELog MAX(MAX(MLFSELog, LLFSELog), OffFSELog) + +#define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */ +/* Each table cannot take more than #symbols * FSELog bits */ +#define ZSTD_MAX_FSE_HEADERS_SIZE (((MaxML + 1) * MLFSELog + (MaxLL + 1) * LLFSELog + (MaxOff + 1) * OffFSELog + 7) / 8) + +static UNUSED_ATTR const U32 LL_bits[MaxLL+1] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, + 4, 6, 7, 8, 9,10,11,12, + 13,14,15,16 +}; +static UNUSED_ATTR const S16 LL_defaultNorm[MaxLL+1] = { + 4, 3, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, + 2, 3, 2, 1, 1, 1, 1, 1, + -1,-1,-1,-1 +}; +#define LL_DEFAULTNORMLOG 6 /* for static allocation */ +static UNUSED_ATTR const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG; + +static UNUSED_ATTR const U32 ML_bits[MaxML+1] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 7, 8, 9,10,11, + 12,13,14,15,16 +}; +static UNUSED_ATTR const S16 ML_defaultNorm[MaxML+1] = { + 1, 4, 3, 2, 2, 2, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1 +}; +#define ML_DEFAULTNORMLOG 6 /* for static allocation */ +static UNUSED_ATTR const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG; + +static UNUSED_ATTR const S16 OF_defaultNorm[DefaultMaxOff+1] = { + 1, 1, 1, 1, 1, 1, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + -1,-1,-1,-1,-1 +}; +#define OF_DEFAULTNORMLOG 5 /* for static allocation */ +static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; + + +/*-******************************************* +* Shared functions to include for inlining +*********************************************/ +static void ZSTD_copy8(void* dst, const void* src) { +#if defined(ZSTD_ARCH_ARM_NEON) + vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src)); +#else + ZSTD_memcpy(dst, src, 8); +#endif +} + +#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } +static void ZSTD_copy16(void* dst, const void* src) { +#if defined(ZSTD_ARCH_ARM_NEON) + vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); +#else + ZSTD_memcpy(dst, src, 16); +#endif +} +#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } + +#define WILDCOPY_OVERLENGTH 32 +#define WILDCOPY_VECLEN 16 + +typedef enum { + ZSTD_no_overlap, + ZSTD_overlap_src_before_dst + /* ZSTD_overlap_dst_before_src, */ +} ZSTD_overlap_e; + +/*! ZSTD_wildcopy() : + * Custom version of ZSTD_memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0) + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart. + * The src buffer must be before the dst buffer. + */ +MEM_STATIC FORCE_INLINE_ATTR +void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype) +{ + ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + length; + + assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff <= -WILDCOPY_VECLEN)); + + if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) { + /* Handle short offset copies. */ + do { + COPY8(op, ip) + } while (op < oend); + } else { + assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first COPY16() call because the copy length is + * almost certain to be short, so the branches have different + * probabilities. Since it is almost certain to be short, only do + * one COPY16() in the first call. Then, do two calls per loop since + * at that point it is more likely to have a high trip count. + */ +#ifdef __aarch64__ + do { + COPY16(op, ip); + } + while (op < oend); +#else + ZSTD_copy16(op, ip); + if (16 >= length) return; + op += 16; + ip += 16; + do { + COPY16(op, ip); + COPY16(op, ip); + } + while (op < oend); +#endif + } +} + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + if (length > 0) { + ZSTD_memcpy(dst, src, length); + } + return length; +} + +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + +/* Controls whether the input/output buffer is buffered or stable. */ +typedef enum { + ZSTD_bm_buffered = 0, /* Buffer the input/output */ + ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ +} ZSTD_bufferMode_e; + + +/*-******************************************* +* Private declarations +*********************************************/ +typedef struct seqDef_s { + U32 offset; /* offset == rawOffset + ZSTD_REP_NUM, or equivalently, offCode + 1 */ + U16 litLength; + U16 matchLength; +} seqDef; + +/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */ +typedef enum { + ZSTD_llt_none = 0, /* no longLengthType */ + ZSTD_llt_literalLength = 1, /* represents a long literal */ + ZSTD_llt_matchLength = 2 /* represents a long match */ +} ZSTD_longLengthType_e; + +typedef struct { + seqDef* sequencesStart; + seqDef* sequences; /* ptr to end of sequences */ + BYTE* litStart; + BYTE* lit; /* ptr to end of literals */ + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; + size_t maxNbSeq; + size_t maxNbLit; + + /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength + * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment + * the existing value of the litLength or matchLength by 0x10000. + */ + ZSTD_longLengthType_e longLengthType; + U32 longLengthPos; /* Index of the sequence to apply long length modification to */ +} seqStore_t; + +typedef struct { + U32 litLength; + U32 matchLength; +} ZSTD_sequenceLength; + +/** + * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences + * indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength. + */ +MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq) +{ + ZSTD_sequenceLength seqLen; + seqLen.litLength = seq->litLength; + seqLen.matchLength = seq->matchLength + MINMATCH; + if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { + seqLen.litLength += 0xFFFF; + } + if (seqStore->longLengthType == ZSTD_llt_matchLength) { + seqLen.matchLength += 0xFFFF; + } + } + return seqLen; +} + +/** + * Contains the compressed frame size and an upper-bound for the decompressed frame size. + * Note: before using `compressedSize`, check for errors using ZSTD_isError(). + * similarly, before using `decompressedBound`, check for errors using: + * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` + */ +typedef struct { + size_t compressedSize; + unsigned long long decompressedBound; +} ZSTD_frameSizeInfo; /* decompress & legacy */ + +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ +void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ + +MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + { +# if defined(_MSC_VER) /* Visual */ +# if STATIC_BMI2 == 1 + return _lzcnt_u32(val)^31; +# else + unsigned long r=0; + return _BitScanReverse(&r, val) ? (unsigned)r : 0; +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ + return __builtin_clz (val) ^ 31; +# elif defined(__ICCARM__) /* IAR Intrinsic */ + return 31 - __CLZ(val); +# else /* Software version */ + static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return DeBruijnClz[(v * 0x07C4ACDDU) >> 27]; +# endif + } +} + +/** + * Computes CTZ on a U64. + * This will be slow on 32-bit mode, and on unsupported compilers. + * If you need this function to be fast (because it is hot) expand + * support. + */ +MEM_STATIC unsigned ZSTD_countTrailingZeros(size_t val) +{ + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 + return _tzcnt_u64(val); +# else + unsigned long r = 0; + return _BitScanForward64( &r, (U64)val ) ? (unsigned)(r >> 3) : 0; +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return __builtin_ctzll((U64)val); +# else + static const int DeBruijnBytePos[64] = { 0, 1, 2, 7, 3, 13, 8, 19, + 4, 25, 14, 28, 9, 34, 20, 56, + 5, 17, 26, 54, 15, 41, 29, 43, + 10, 31, 38, 35, 21, 45, 49, 57, + 63, 6, 12, 18, 24, 27, 33, 55, + 16, 53, 40, 42, 30, 37, 44, 48, + 62, 11, 23, 32, 52, 39, 36, 47, + 61, 22, 51, 46, 60, 50, 59, 58 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + unsigned long r=0; + return _BitScanForward( &r, (U32)val ) ? (unsigned)(r >> 3) : 0; +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 1, 28, 2, 29, 14, 24, 3, + 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, + 26, 12, 18, 6, 11, 5, 10, 9 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } +} + + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ + + +typedef struct { + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; /* declared here for decompress and fullbench */ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr); + +/*! ZSTD_decodeSeqHeaders() : + * decode sequence header from src */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize); + +/* + * Custom memory allocation functions + */ +typedef struct { void* customAlloc; void* customFree; void* opaque; } ZSTD_customMem; +extern ZSTD_customMem ZSTD_defaultCMem; +static inline void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) { + (void)customMem; + return malloc(size); +} +static inline void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) { + (void)customMem; + free(ptr); +} + +#endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/src/bled/zstd_mem.h b/src/bled/zstd_mem.h new file mode 100644 index 00000000000..b5d15f1abb6 --- /dev/null +++ b/src/bled/zstd_mem.h @@ -0,0 +1,423 @@ +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + + +/*-**************************************** +* Dependencies +******************************************/ +#include <stddef.h> /* size_t, ptrdiff_t */ +#include "zstd_compiler.h" /* __has_builtin */ +//#include "debug.h" /* DEBUG_STATIC_ASSERT */ +#include "zstd_deps.h" /* ZSTD_memcpy */ + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include <stdlib.h> /* _byteswap_ulong */ +# include <intrin.h> /* _byteswap_* */ +#endif + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# if defined(_AIX) +# include <inttypes.h> +# else +# include <stdint.h> /* intptr_t */ +# endif + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else +# include <limits.h> +#if CHAR_BIT != 8 +# error "this implementation requires char to be exactly 8-bit type" +#endif + typedef unsigned char BYTE; +#if USHRT_MAX != 65535 +# error "this implementation requires short to be exactly 16-bit type" +#endif + typedef unsigned short U16; + typedef signed short S16; +#if UINT_MAX != 4294967295 +# error "this implementation requires int to be exactly 32-bit type" +#endif + typedef unsigned int U32; + typedef signed int S32; +/* note : there are no limits defined for long long type in C90. + * limits exist in C99, however, in such case, <stdint.h> is preferred */ + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + +/*-************************************************************** +* Memory I/O API +*****************************************************************/ +/*=== Static platform detection ===*/ +MEM_STATIC unsigned MEM_32bits(void); +MEM_STATIC unsigned MEM_64bits(void); +MEM_STATIC unsigned MEM_isLittleEndian(void); + +/*=== Native unaligned read/write ===*/ +MEM_STATIC U16 MEM_read16(const void* memPtr); +MEM_STATIC U32 MEM_read32(const void* memPtr); +MEM_STATIC U64 MEM_read64(const void* memPtr); +MEM_STATIC size_t MEM_readST(const void* memPtr); + +MEM_STATIC void MEM_write16(void* memPtr, U16 value); +MEM_STATIC void MEM_write32(void* memPtr, U32 value); +MEM_STATIC void MEM_write64(void* memPtr, U64 value); + +/*=== Little endian unaligned read/write ===*/ +MEM_STATIC U16 MEM_readLE16(const void* memPtr); +MEM_STATIC U32 MEM_readLE24(const void* memPtr); +MEM_STATIC U32 MEM_readLE32(const void* memPtr); +MEM_STATIC U64 MEM_readLE64(const void* memPtr); +MEM_STATIC size_t MEM_readLEST(const void* memPtr); + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val); +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val); +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val); + +/*=== Big endian unaligned read/write ===*/ +MEM_STATIC U32 MEM_readBE32(const void* memPtr); +MEM_STATIC U64 MEM_readBE64(const void* memPtr); +MEM_STATIC size_t MEM_readBEST(const void* memPtr); + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val); + +/*=== Byteswap ===*/ +MEM_STATIC U32 MEM_swap32(U32 in); +MEM_STATIC U64 MEM_swap64(U64 in); +MEM_STATIC size_t MEM_swapST(size_t in); + + +/*-************************************************************** +* Memory I/O Implementation +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6) + * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + return 1; +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + return 0; +#elif defined(__clang__) && __LITTLE_ENDIAN__ + return 1; +#elif defined(__clang__) && __BIG_ENDIAN__ + return 0; +#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86) + return 1; +#elif defined(__DMC__) && defined(_M_IX86) + return 1; +#else + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +#endif +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } +MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) + __pragma( pack(push, 1) ) + typedef struct { U16 v; } unalign16; + typedef struct { U32 v; } unalign32; + typedef struct { U64 v; } unalign64; + typedef struct { size_t v; } unalignArch; + __pragma( pack(pop) ) +#else + typedef struct { U16 v; } __attribute__((packed)) unalign16; + typedef struct { U32 v; } __attribute__((packed)) unalign32; + typedef struct { U64 v; } __attribute__((packed)) unalign64; + typedef struct { size_t v; } __attribute__((packed)) unalignArch; +#endif + +MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; } +MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap32)) + return __builtin_bswap32(in); +#else + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +#endif +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap64)) + return __builtin_bswap64(in); +#else + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val>>16); +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + +/* code only tested on 32 and 64 bits systems */ +MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + + + +#endif /* MEM_H_MODULE */ diff --git a/src/rufus.c b/src/rufus.c index b343c0c9251..fa06a813b94 100755 --- a/src/rufus.c +++ b/src/rufus.c @@ -2538,7 +2538,7 @@ static INT_PTR CALLBACK MainCallback(HWND hDlg, UINT message, WPARAM wParam, LPA img_provided = FALSE; // One off thing... } else { char* old_image_path = image_path; - char extensions[128] = "*.iso;*.img;*.vhd;*.vhdx;*.usb;*.bz2;*.bzip2;*.gz;*.lzma;*.xz;*.Z;*.zip;*.wic;*.wim;*.esd;*.vtsi"; + char extensions[128] = "*.iso;*.img;*.vhd;*.vhdx;*.usb;*.bz2;*.bzip2;*.gz;*.lzma;*.xz;*.Z;*.zip;*.zst;*.wic;*.wim;*.esd;*.vtsi"; if (has_ffu_support) strcat(extensions, ";*.ffu"); // If declared globaly, lmprintf(MSG_280) would be called on each message... diff --git a/src/rufus.rc b/src/rufus.rc index a391c4f5de7..59fd437a314 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.7.2210" +CAPTION "Rufus 4.7.2211" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -399,8 +399,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,7,2210,0 - PRODUCTVERSION 4,7,2210,0 + FILEVERSION 4,7,2211,0 + PRODUCTVERSION 4,7,2211,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -418,13 +418,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.7.2210" + VALUE "FileVersion", "4.7.2211" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", " 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.7.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.7.2210" + VALUE "ProductVersion", "4.7.2211" END END BLOCK "VarFileInfo" diff --git a/src/vhd.c b/src/vhd.c index a184b2f63e1..eef595bc223 100644 --- a/src/vhd.c +++ b/src/vhd.c @@ -104,6 +104,7 @@ static comp_assoc file_assoc[] = { { ".bz2", BLED_COMPRESSION_BZIP2 }, { ".xz", BLED_COMPRESSION_XZ }, { ".vtsi", BLED_COMPRESSION_VTSI }, + { ".zst", BLED_COMPRESSION_ZSTD }, { ".ffu", BLED_COMPRESSION_MAX }, { ".vhd", BLED_COMPRESSION_MAX + 1 }, { ".vhdx", BLED_COMPRESSION_MAX + 2 }, From 8f3a9c1c7c6c1538a4d65ec15d06bc199aafe115 Mon Sep 17 00:00:00 2001 From: Pete Batard <pete@akeo.ie> Date: Wed, 27 Nov 2024 17:19:36 +0000 Subject: [PATCH 30/30] [cmp] update Bled to latest * This mostly updates the ZSTD code to latest and properly removes all debug message. * Also switch MinGW from gnu99 to gnu11. --- .vs/bled.vcxproj | 3 + .vs/bled.vcxproj.filters | 9 + configure | 2 +- configure.ac | 2 +- src/bled/Makefile.am | 3 +- src/bled/Makefile.in | 17 +- src/bled/data_align.c | 1 - src/bled/data_skip.c | 1 - src/bled/decompress_bunzip2.c | 2 +- src/bled/decompress_gunzip.c | 1 - src/bled/decompress_unzip.c | 2 - src/bled/decompress_unzstd.c | 6 - src/bled/filter_accept_all.c | 1 - src/bled/filter_accept_list.c | 1 - src/bled/find_list_entry.c | 1 - src/bled/fse.h | 126 +- src/bled/fse_bitstream.h | 156 +- src/bled/fse_decompress.c | 134 +- src/bled/header_verbose_list.c | 2 +- src/bled/huf.h | 250 +- src/bled/huf_decompress.c | 925 ++-- src/bled/init_handle.c | 1 - src/bled/libbb.h | 66 +- src/bled/open_transformer.c | 132 +- src/bled/seek_by_jump.c | 1 - src/bled/seek_by_read.c | 1 - src/bled/xxhash.c | 815 +-- src/bled/xxhash.h | 7118 ++++++++++++++++++++++++++- src/bled/xz_dec_lzma2.c | 8 +- src/bled/zstd.h | 3071 +++++++++++- src/bled/zstd_bits.h | 208 + src/bled/zstd_common.c | 48 + src/bled/zstd_compiler.h | 319 +- src/bled/zstd_config.h | 11 + src/bled/zstd_cpu.h | 38 +- src/bled/zstd_ddict.c | 244 + src/bled/zstd_ddict.h | 15 +- src/bled/zstd_decompress.c | 1336 ++++- src/bled/zstd_decompress_block.c | 1255 ++++- src/bled/zstd_decompress_block.h | 17 +- src/bled/zstd_decompress_internal.h | 53 +- src/bled/zstd_deps.h | 36 +- src/bled/zstd_entropy_common.c | 54 +- src/bled/zstd_error_private.c | 11 +- src/bled/zstd_error_private.h | 98 +- src/bled/zstd_errors.h | 49 +- src/bled/zstd_internal.h | 180 +- src/bled/zstd_mem.h | 103 +- src/rufus.rc | 10 +- 49 files changed, 14322 insertions(+), 2621 deletions(-) create mode 100644 src/bled/zstd_bits.h create mode 100644 src/bled/zstd_common.c create mode 100644 src/bled/zstd_ddict.c diff --git a/.vs/bled.vcxproj b/.vs/bled.vcxproj index 2c25bf54449..12050177630 100644 --- a/.vs/bled.vcxproj +++ b/.vs/bled.vcxproj @@ -65,6 +65,8 @@ <ClCompile Include="..\src\bled\xz_dec_bcj.c" /> <ClCompile Include="..\src\bled\xz_dec_lzma2.c" /> <ClCompile Include="..\src\bled\xz_dec_stream.c" /> + <ClCompile Include="..\src\bled\zstd_common.c" /> + <ClCompile Include="..\src\bled\zstd_ddict.c" /> <ClCompile Include="..\src\bled\zstd_decompress.c" /> <ClCompile Include="..\src\bled\zstd_decompress_block.c" /> <ClCompile Include="..\src\bled\zstd_entropy_common.c" /> @@ -85,6 +87,7 @@ <ClInclude Include="..\src\bled\xz_private.h" /> <ClInclude Include="..\src\bled\xz_stream.h" /> <ClInclude Include="..\src\bled\zstd.h" /> + <ClInclude Include="..\src\bled\zstd_bits.h" /> <ClInclude Include="..\src\bled\zstd_compiler.h" /> <ClInclude Include="..\src\bled\zstd_config.h" /> <ClInclude Include="..\src\bled\zstd_cpu.h" /> diff --git a/.vs/bled.vcxproj.filters b/.vs/bled.vcxproj.filters index aec232074bc..abbc6ed8745 100644 --- a/.vs/bled.vcxproj.filters +++ b/.vs/bled.vcxproj.filters @@ -111,6 +111,12 @@ <ClCompile Include="..\src\bled\zstd_error_private.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\src\bled\zstd_common.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\src\bled\zstd_ddict.c"> + <Filter>Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <ClInclude Include="..\src\bled\bb_archive.h"> @@ -188,5 +194,8 @@ <ClInclude Include="..\src\bled\zstd_mem.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\src\bled\zstd_bits.h"> + <Filter>Header Files</Filter> + </ClInclude> </ItemGroup> </Project> \ No newline at end of file diff --git a/configure b/configure index a385d034024..69e58bcdd8f 100755 --- a/configure +++ b/configure @@ -4725,7 +4725,7 @@ fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS="${saved_CFLAGS}" -AM_CFLAGS="$AM_CFLAGS -DUNICODE -D_UNICODE -UNDEBUG -DCOBJMACROS -D__USE_MINGW_ANSI_STDIO=0 -std=gnu99 -Wshadow -Wall -Wformat-security -Wundef -Wunused -Wstrict-prototypes -Wno-restrict -Wno-array-bounds -Werror-implicit-function-declaration -Wbidi-chars=none $nopointersign_cflags" +AM_CFLAGS="$AM_CFLAGS -DUNICODE -D_UNICODE -UNDEBUG -DCOBJMACROS -D__USE_MINGW_ANSI_STDIO=0 -std=gnu11 -Wshadow -Wall -Wformat-security -Wundef -Wunused -Wstrict-prototypes -Wno-restrict -Wno-array-bounds -Werror-implicit-function-declaration -Wbidi-chars=none $nopointersign_cflags" diff --git a/configure.ac b/configure.ac index c530cd1010d..27ccdbfaf5f 100644 --- a/configure.ac +++ b/configure.ac @@ -64,7 +64,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([])], [nopointersign_cflags="-Wno-pointer-sign"], [nopointersign_cflags=""]) CFLAGS="${saved_CFLAGS}" -AM_CFLAGS="$AM_CFLAGS -DUNICODE -D_UNICODE -UNDEBUG -DCOBJMACROS -D__USE_MINGW_ANSI_STDIO=0 -std=gnu99 -Wshadow -Wall -Wformat-security -Wundef -Wunused -Wstrict-prototypes -Wno-restrict -Wno-array-bounds -Werror-implicit-function-declaration -Wbidi-chars=none $nopointersign_cflags" +AM_CFLAGS="$AM_CFLAGS -DUNICODE -D_UNICODE -UNDEBUG -DCOBJMACROS -D__USE_MINGW_ANSI_STDIO=0 -std=gnu11 -Wshadow -Wall -Wformat-security -Wundef -Wunused -Wstrict-prototypes -Wno-restrict -Wno-array-bounds -Werror-implicit-function-declaration -Wbidi-chars=none $nopointersign_cflags" AC_SUBST([VISIBILITY_CFLAGS]) AC_SUBST([AM_CFLAGS]) diff --git a/src/bled/Makefile.am b/src/bled/Makefile.am index 07417ca20d9..23b00401b7c 100644 --- a/src/bled/Makefile.am +++ b/src/bled/Makefile.am @@ -5,5 +5,6 @@ libbled_a_SOURCES = bled.c crc32.c data_align.c data_extract_all.c data_skip.c d decompress_unzstd.c decompress_vtsi.c filter_accept_all.c filter_accept_list.c filter_accept_reject_list.c \ find_list_entry.c fse_decompress.c header_list.c header_skip.c header_verbose_list.c huf_decompress.c \ init_handle.c open_transformer.c seek_by_jump.c seek_by_read.c xz_dec_bcj.c xz_dec_lzma2.c xz_dec_stream.c \ - xxhash.c zstd_decompress.c zstd_decompress_block.c zstd_entropy_common.c zstd_error_private.c + xxhash.c zstd_common.c zstd_decompress.c zstd_decompress_block.c zstd_ddict.c zstd_entropy_common.c \ + zstd_error_private.c libbled_a_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/.. -Wno-undef -Wno-strict-aliasing diff --git a/src/bled/Makefile.in b/src/bled/Makefile.in index cc7463e3c46..7619dc506f9 100644 --- a/src/bled/Makefile.in +++ b/src/bled/Makefile.in @@ -121,8 +121,10 @@ am_libbled_a_OBJECTS = libbled_a-bled.$(OBJEXT) \ libbled_a-xz_dec_bcj.$(OBJEXT) \ libbled_a-xz_dec_lzma2.$(OBJEXT) \ libbled_a-xz_dec_stream.$(OBJEXT) libbled_a-xxhash.$(OBJEXT) \ + libbled_a-zstd_common.$(OBJEXT) \ libbled_a-zstd_decompress.$(OBJEXT) \ libbled_a-zstd_decompress_block.$(OBJEXT) \ + libbled_a-zstd_ddict.$(OBJEXT) \ libbled_a-zstd_entropy_common.$(OBJEXT) \ libbled_a-zstd_error_private.$(OBJEXT) libbled_a_OBJECTS = $(am_libbled_a_OBJECTS) @@ -279,7 +281,8 @@ libbled_a_SOURCES = bled.c crc32.c data_align.c data_extract_all.c data_skip.c d decompress_unzstd.c decompress_vtsi.c filter_accept_all.c filter_accept_list.c filter_accept_reject_list.c \ find_list_entry.c fse_decompress.c header_list.c header_skip.c header_verbose_list.c huf_decompress.c \ init_handle.c open_transformer.c seek_by_jump.c seek_by_read.c xz_dec_bcj.c xz_dec_lzma2.c xz_dec_stream.c \ - xxhash.c zstd_decompress.c zstd_decompress_block.c zstd_entropy_common.c zstd_error_private.c + xxhash.c zstd_common.c zstd_decompress.c zstd_decompress_block.c zstd_ddict.c zstd_entropy_common.c \ + zstd_error_private.c libbled_a_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/.. -Wno-undef -Wno-strict-aliasing all: all-am @@ -517,6 +520,12 @@ libbled_a-xxhash.o: xxhash.c libbled_a-xxhash.obj: xxhash.c $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-xxhash.obj `if test -f 'xxhash.c'; then $(CYGPATH_W) 'xxhash.c'; else $(CYGPATH_W) '$(srcdir)/xxhash.c'; fi` +libbled_a-zstd_common.o: zstd_common.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_common.o `test -f 'zstd_common.c' || echo '$(srcdir)/'`zstd_common.c + +libbled_a-zstd_common.obj: zstd_common.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_common.obj `if test -f 'zstd_common.c'; then $(CYGPATH_W) 'zstd_common.c'; else $(CYGPATH_W) '$(srcdir)/zstd_common.c'; fi` + libbled_a-zstd_decompress.o: zstd_decompress.c $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_decompress.o `test -f 'zstd_decompress.c' || echo '$(srcdir)/'`zstd_decompress.c @@ -529,6 +538,12 @@ libbled_a-zstd_decompress_block.o: zstd_decompress_block.c libbled_a-zstd_decompress_block.obj: zstd_decompress_block.c $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_decompress_block.obj `if test -f 'zstd_decompress_block.c'; then $(CYGPATH_W) 'zstd_decompress_block.c'; else $(CYGPATH_W) '$(srcdir)/zstd_decompress_block.c'; fi` +libbled_a-zstd_ddict.o: zstd_ddict.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_ddict.o `test -f 'zstd_ddict.c' || echo '$(srcdir)/'`zstd_ddict.c + +libbled_a-zstd_ddict.obj: zstd_ddict.c + $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_ddict.obj `if test -f 'zstd_ddict.c'; then $(CYGPATH_W) 'zstd_ddict.c'; else $(CYGPATH_W) '$(srcdir)/zstd_ddict.c'; fi` + libbled_a-zstd_entropy_common.o: zstd_entropy_common.c $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libbled_a_CFLAGS) $(CFLAGS) -c -o libbled_a-zstd_entropy_common.o `test -f 'zstd_entropy_common.c' || echo '$(srcdir)/'`zstd_entropy_common.c diff --git a/src/bled/data_align.c b/src/bled/data_align.c index a6b84a440f0..f61fdd93fda 100644 --- a/src/bled/data_align.c +++ b/src/bled/data_align.c @@ -2,7 +2,6 @@ /* * Licensed under GPLv2 or later, see file LICENSE in this source tree. */ - #include "libbb.h" #include "bb_archive.h" diff --git a/src/bled/data_skip.c b/src/bled/data_skip.c index 588167f01cb..1a608227e06 100644 --- a/src/bled/data_skip.c +++ b/src/bled/data_skip.c @@ -2,7 +2,6 @@ /* * Licensed under GPLv2 or later, see file LICENSE in this source tree. */ - #include "libbb.h" #include "bb_archive.h" diff --git a/src/bled/decompress_bunzip2.c b/src/bled/decompress_bunzip2.c index 39ea43e7757..3c54285d7f7 100644 --- a/src/bled/decompress_bunzip2.c +++ b/src/bled/decompress_bunzip2.c @@ -163,7 +163,7 @@ static int get_next_block(bunzip_data *bd) uint32_t *dbuf; unsigned origPtr, t; unsigned dbufCount, runPos; - unsigned runCnt = 0; // runCnt; /* for compiler */ + unsigned runCnt = 0; /* for compiler */ dbuf = bd->dbuf; selectors = bd->selectors; diff --git a/src/bled/decompress_gunzip.c b/src/bled/decompress_gunzip.c index 6f0ddcf92f0..beb7fb39138 100644 --- a/src/bled/decompress_gunzip.c +++ b/src/bled/decompress_gunzip.c @@ -1269,7 +1269,6 @@ unpack_gz_stream(transformer_state_t *xstate) return -1; } to_read = -1; -// bytebuffer_max = 0x8000; bytebuffer = xmalloc(bytebuffer_max); if (bytebuffer == NULL) { bb_error_msg("alloc error"); diff --git a/src/bled/decompress_unzip.c b/src/bled/decompress_unzip.c index ad698352b50..ef8745a2b55 100644 --- a/src/bled/decompress_unzip.c +++ b/src/bled/decompress_unzip.c @@ -17,8 +17,6 @@ # define dbg(...) ((void)0) #endif -#define xread safe_read - enum { #if BB_BIG_ENDIAN ZIP_FILEHEADER_MAGIC = 0x504b0304, diff --git a/src/bled/decompress_unzstd.c b/src/bled/decompress_unzstd.c index 1792571fdd6..894ad5cbc89 100644 --- a/src/bled/decompress_unzstd.c +++ b/src/bled/decompress_unzstd.c @@ -17,12 +17,6 @@ #include "zstd_deps.h" #include "zstd_internal.h" -ZSTD_customMem ZSTD_defaultCMem = { ZSTD_customMalloc, ZSTD_customFree, NULL }; - -ZSTDLIB_API const char* ZSTD_getErrorName(size_t code) { - return ERR_getErrorName(code); -} - ALWAYS_INLINE static size_t roundupsize(size_t size, size_t align) { return (size + align - 1U) & ~(align - 1); diff --git a/src/bled/filter_accept_all.c b/src/bled/filter_accept_all.c index c33f7d3e368..5a0ee9c605f 100644 --- a/src/bled/filter_accept_all.c +++ b/src/bled/filter_accept_all.c @@ -4,7 +4,6 @@ * * Licensed under GPLv2 or later, see file LICENSE in this source tree. */ - #include "libbb.h" #include "bb_archive.h" diff --git a/src/bled/filter_accept_list.c b/src/bled/filter_accept_list.c index a2d4b23e982..32f80657450 100644 --- a/src/bled/filter_accept_list.c +++ b/src/bled/filter_accept_list.c @@ -4,7 +4,6 @@ * * Licensed under GPLv2 or later, see file LICENSE in this source tree. */ - #include "libbb.h" #include "bb_archive.h" diff --git a/src/bled/find_list_entry.c b/src/bled/find_list_entry.c index 21034dbf7ce..48f3462d32a 100644 --- a/src/bled/find_list_entry.c +++ b/src/bled/find_list_entry.c @@ -5,7 +5,6 @@ * Licensed under GPLv2 or later, see file LICENSE in this source tree. */ -//#include <fnmatch.h> #include "libbb.h" #include "bb_archive.h" diff --git a/src/bled/fse.h b/src/bled/fse.h index 9bf7cb43cc6..71015df44dd 100644 --- a/src/bled/fse.h +++ b/src/bled/fse.h @@ -1,7 +1,7 @@ /* ****************************************************************** * FSE : Finite State Entropy codec * Public Prototypes declaration - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -12,6 +12,9 @@ * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ +#if defined (__cplusplus) +extern "C" { +#endif #ifndef FSE_H #define FSE_H @@ -23,7 +26,18 @@ #include "zstd_deps.h" /* size_t, ptrdiff_t */ -# define FSE_PUBLIC_API MEM_STATIC +/*-***************************************** +* FSE_PUBLIC_API : control library symbols visibility +******************************************/ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define FSE_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define FSE_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define FSE_PUBLIC_API +#endif /*------ Version ------*/ #define FSE_VERSION_MAJOR 0 @@ -39,58 +53,16 @@ FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ -/*-**************************************** -* FSE simple functions -******************************************/ -/*! FSE_compress() : - Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. - 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). - @return : size of compressed data (<= dstCapacity). - Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. - if FSE_isError(return), compression failed (more details using FSE_getErrorName()) -*/ -FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/*! FSE_decompress(): - Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', - into already allocated destination buffer 'dst', of size 'dstCapacity'. - @return : size of regenerated data (<= maxDstSize), - or an error code, which can be tested using FSE_isError() . - - ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! - Why ? : making this distinction requires a header. - Header management is intentionally delegated to the user layer, which can better manage special cases. -*/ -FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, - const void* cSrc, size_t cSrcSize); - - /*-***************************************** * Tool functions ******************************************/ FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ /* Error Management */ -unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ -/*-***************************************** -* FSE advanced functions -******************************************/ -/*! FSE_compress2() : - Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' - Both parameters can be defined as '0' to mean : use default value - @return : size of compressed data - Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. - if FSE_isError(return), it's an error code. -*/ -FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); - - /*-***************************************** * FSE detailed API ******************************************/ @@ -150,8 +122,6 @@ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, /*! Constructor and Destructor of FSE_CTable. Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ -FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog); -FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct); /*! FSE_buildCTable(): Builds `ct`, which must be already allocated, using FSE_createCTable(). @@ -216,34 +186,18 @@ If there is an error, the function will return an ErrorCode (which can be tested @return : size read from 'rBuffer', or an errorCode, which can be tested using FSE_isError(). maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ -size_t FSE_readNCount (short* normalizedCounter, +FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); /*! FSE_readNCount_bmi2(): * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise. */ -size_t FSE_readNCount_bmi2(short* normalizedCounter, +FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize, int bmi2); -/*! Constructor and Destructor of FSE_DTable. - Note that its size depends on 'tableLog' */ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ -FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); -FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); - -/*! FSE_buildDTable(): - Builds 'dt', which must be already allocated, using FSE_createDTable(). - return : 0, or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -/*! FSE_decompress_usingDTable(): - Decompress compressed source `cSrc` of size `cSrcSize` using `dt` - into `dst` which must be already allocated. - @return : size of regenerated data (necessarily <= `dstCapacity`), - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); /*! Tutorial : @@ -275,7 +229,8 @@ If there is an error, the function will return an error code, which can be teste #endif /* FSE_H */ -#if !defined(FSE_H_FSE_STATIC_LINKING_ONLY) + +#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY) #define FSE_H_FSE_STATIC_LINKING_ONLY /* *** Dependency *** */ @@ -303,20 +258,10 @@ If there is an error, the function will return an error code, which can be teste * FSE advanced API ***************************************** */ -MEM_STATIC unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); /**< same as FSE_optimalTableLog(), which used `minus==2` */ -/* FSE_compress_wksp() : - * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). - * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. - */ -#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) -MEM_STATIC size_t FSE_compress_wksp(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); - -MEM_STATIC size_t FSE_buildCTable_raw(FSE_CTable* ct, unsigned nbBits); -/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ - -MEM_STATIC size_t FSE_buildCTable_rle(FSE_CTable* ct, unsigned char symbolValue); +size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); /**< build a fake FSE_CTable, designed to compress always the same symbolValue */ /* FSE_buildCTable_wksp() : @@ -326,26 +271,18 @@ MEM_STATIC size_t FSE_buildCTable_rle(FSE_CTable* ct, unsigned char symbolValue) */ #define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */) #define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)) -MEM_STATIC size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); +size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); #define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8) #define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned)) FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ -MEM_STATIC size_t FSE_buildDTable_raw(FSE_DTable* dt, unsigned nbBits); -/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */ - -MEM_STATIC size_t FSE_buildDTable_rle(FSE_DTable* dt, unsigned char symbolValue); -/**< build a fake FSE_DTable, designed to always generate the same symbolValue */ - -#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) #define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) -MEM_STATIC size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize); -/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */ - size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); -/**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */ +/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`. + * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */ typedef enum { FSE_repeat_none, /**< Cannot use the previous table */ @@ -528,20 +465,20 @@ MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, un FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; const U16* const stateTable = (const U16*)(statePtr->stateTable); U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); - BIT_addBits(bitC, statePtr->value, nbBitsOut); + BIT_addBits(bitC, (size_t)statePtr->value, nbBitsOut); statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; } MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) { - BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_addBits(bitC, (size_t)statePtr->value, statePtr->stateLog); BIT_flushBits(bitC); } /* FSE_getMaxNbBits() : * Approximate maximum cost of a symbol, in bits. - * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) * note 1 : assume symbolValue is valid (<= maxSymbolValue) * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) @@ -698,3 +635,6 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) #endif /* FSE_STATIC_LINKING_ONLY */ +#if defined (__cplusplus) +} +#endif diff --git a/src/bled/fse_bitstream.h b/src/bled/fse_bitstream.h index a6f3622923a..ed57e32b6e7 100644 --- a/src/bled/fse_bitstream.h +++ b/src/bled/fse_bitstream.h @@ -1,7 +1,7 @@ /* ****************************************************************** * bitstream * Part of FSE library - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -14,6 +14,9 @@ #ifndef BITSTREAM_H_MODULE #define BITSTREAM_H_MODULE +#if defined (__cplusplus) +extern "C" { +#endif /* * This API consists of small unitary functions, which must be inlined for best performance. * Since link-time-optimization is not available for all compilers, @@ -26,11 +29,19 @@ #include "zstd_mem.h" /* unaligned access routines */ #include "zstd_compiler.h" /* UNLIKELY() */ #include "zstd_error_private.h" /* error codes and messages */ +#include "zstd_bits.h" /* ZSTD_highbit32 */ /*========================================= * Target specific =========================================*/ +#ifndef ZSTD_NO_INTRINSICS +# if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__) +# include <immintrin.h> /* support for bextr (experimental)/bzhi */ +# elif defined(__ICCARM__) +# include <intrinsics.h> +# endif +#endif #define STREAM_ACCUMULATOR_MIN_32 25 #define STREAM_ACCUMULATOR_MIN_64 57 @@ -78,19 +89,20 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); /*-******************************************** * bitStream decoding API (read backward) **********************************************/ +typedef size_t BitContainerType; typedef struct { - size_t bitContainer; + BitContainerType bitContainer; unsigned bitsConsumed; const char* ptr; const char* start; const char* limitPtr; } BIT_DStream_t; -typedef enum { BIT_DStream_unfinished = 0, - BIT_DStream_endOfBuffer = 1, - BIT_DStream_completed = 2, - BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ - /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ +typedef enum { BIT_DStream_unfinished = 0, /* fully refilled */ + BIT_DStream_endOfBuffer = 1, /* still some bits left in bitstream */ + BIT_DStream_completed = 2, /* bitstream entirely consumed, bit-exact */ + BIT_DStream_overflow = 3 /* user requested more bits than present in bitstream */ + } BIT_DStream_status; /* result of BIT_reloadDStream() */ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); @@ -100,7 +112,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); /* Start by invoking BIT_initDStream(). * A chunk of the bitStream is then stored into a local register. -* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (BitContainerType). * You can then retrieve bitFields stored into the local register, **in reverse order**. * Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. * A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. @@ -121,42 +133,6 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); /* faster, but works only if nbBits >= 1 */ - - -/*-************************************************************** -* Internal functions -****************************************************************/ -MEM_STATIC unsigned BIT_highbit32 (U32 val) -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ -# if STATIC_BMI2 == 1 - return _lzcnt_u32(val) ^ 31; -# else - unsigned long r = 0; - return _BitScanReverse(&r, val) ? (unsigned)r : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return __builtin_clz (val) ^ 31; -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return 31 - __CLZ(val); -# else /* Software version */ - static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, - 11, 14, 16, 18, 22, 25, 3, 30, - 8, 12, 20, 28, 15, 17, 24, 7, - 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; -# endif - } -} - /*===== Local Constants =====*/ static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, @@ -186,6 +162,16 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, return 0; } +FORCE_INLINE_TEMPLATE size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ +#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS) + return _bzhi_u64(bitContainer, nbBits); +#else + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +#endif +} + /*! BIT_addBits() : * can add up to 31 bits into `bitC`. * Note : does not check for register overflow ! */ @@ -195,7 +181,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); assert(nbBits < BIT_MASK_SIZE); assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); - bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; + bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos; bitC->bitPos += nbBits; } @@ -274,35 +260,35 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); bitD->bitContainer = MEM_readLEST(bitD->ptr); { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } } else { bitD->ptr = bitD->start; bitD->bitContainer = *(const BYTE*)(bitD->start); switch(srcSize) { - case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + case 7: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); ZSTD_FALLTHROUGH; - case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + case 6: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); ZSTD_FALLTHROUGH; - case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + case 5: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); ZSTD_FALLTHROUGH; - case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + case 4: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[3]) << 24; ZSTD_FALLTHROUGH; - case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + case 3: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[2]) << 16; ZSTD_FALLTHROUGH; - case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; + case 2: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[1]) << 8; ZSTD_FALLTHROUGH; default: break; } { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ } bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; @@ -311,12 +297,12 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si return srcSize; } -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +FORCE_INLINE_TEMPLATE size_t BIT_getUpperBits(BitContainerType bitContainer, U32 const start) { return bitContainer >> start; } -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +FORCE_INLINE_TEMPLATE size_t BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits) { U32 const regMask = sizeof(bitContainer)*8 - 1; /* if start > regMask, bitstream is corrupted, and result is undefined */ @@ -333,23 +319,13 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 c #endif } -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) -{ -#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 - return _bzhi_u64(bitContainer, nbBits); -#else - assert(nbBits < BIT_MASK_SIZE); - return bitContainer & BIT_mask[nbBits]; -#endif -} - /*! BIT_lookBits() : * Provides next n bits from local register. * local register is not modified. * On 32-bits, maxNbBits==24. * On 64-bits, maxNbBits==56. * @return : value extracted */ -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +FORCE_INLINE_TEMPLATE size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) { /* arbitrate between double-shift and shift+mask */ #if 1 @@ -372,7 +348,7 @@ MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); } -MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) { bitD->bitsConsumed += nbBits; } @@ -381,7 +357,7 @@ MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) * Read (consume) next n bits from local register and update. * Pay attention to not read more than nbBits contained into local register. * @return : extracted value. */ -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +FORCE_INLINE_TEMPLATE size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) { size_t const value = BIT_lookBits(bitD, nbBits); BIT_skipBits(bitD, nbBits); @@ -389,7 +365,7 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned n } /*! BIT_readBitsFast() : - * unsafe version; only works only if nbBits >= 1 */ + * unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) { size_t const value = BIT_lookBitsFast(bitD, nbBits); @@ -398,6 +374,21 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) return value; } +/*! BIT_reloadDStream_internal() : + * Simple variant of BIT_reloadDStream(), with two conditions: + * 1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8 + * 2. look window is valid after shifted down : bitD->ptr >= bitD->start + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream_internal(BIT_DStream_t* bitD) +{ + assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); + bitD->ptr -= bitD->bitsConsumed >> 3; + assert(bitD->ptr >= bitD->start); + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; +} + /*! BIT_reloadDStreamFast() : * Similar to BIT_reloadDStream(), but with two differences: * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! @@ -408,31 +399,35 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) { if (UNLIKELY(bitD->ptr < bitD->limitPtr)) return BIT_DStream_overflow; - assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); - bitD->ptr -= bitD->bitsConsumed >> 3; - bitD->bitsConsumed &= 7; - bitD->bitContainer = MEM_readLEST(bitD->ptr); - return BIT_DStream_unfinished; + return BIT_reloadDStream_internal(bitD); } /*! BIT_reloadDStream() : * Refill `bitD` from buffer previously set in BIT_initDStream() . - * This function is safe, it guarantees it will not read beyond src buffer. + * This function is safe, it guarantees it will not never beyond src buffer. * @return : status of `BIT_DStream_t` internal register. * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ -MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) { - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ + /* note : once in overflow mode, a bitstream remains in this mode until it's reset */ + if (UNLIKELY(bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))) { + static const BitContainerType zeroFilled = 0; + bitD->ptr = (const char*)&zeroFilled; /* aliasing is allowed for char */ + /* overflow detected, erroneous scenario or end of stream: no update */ return BIT_DStream_overflow; + } + + assert(bitD->ptr >= bitD->start); if (bitD->ptr >= bitD->limitPtr) { - return BIT_reloadDStreamFast(bitD); + return BIT_reloadDStream_internal(bitD); } if (bitD->ptr == bitD->start) { + /* reached end of bitStream => no update */ if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; return BIT_DStream_completed; } - /* start < ptr < limitPtr */ + /* start < ptr < limitPtr => cautious update */ { U32 nbBytes = bitD->bitsConsumed >> 3; BIT_DStream_status result = BIT_DStream_unfinished; if (bitD->ptr - nbBytes < bitD->start) { @@ -454,5 +449,8 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); } +#if defined (__cplusplus) +} +#endif #endif /* BITSTREAM_H_MODULE */ diff --git a/src/bled/fse_decompress.c b/src/bled/fse_decompress.c index 610898e8683..63e478e541d 100644 --- a/src/bled/fse_decompress.c +++ b/src/bled/fse_decompress.c @@ -1,6 +1,6 @@ /* ****************************************************************** * FSE : Finite State Entropy decoder - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -16,14 +16,13 @@ /* ************************************************************** * Includes ****************************************************************/ -//#include "debug.h" /* assert */ #include "fse_bitstream.h" #include "zstd_compiler.h" #define FSE_STATIC_LINKING_ONLY #include "fse.h" #include "zstd_error_private.h" -#define ZSTD_DEPS_NEED_MALLOC -#include "zstd_deps.h" +#include "zstd_deps.h" /* ZSTD_memcpy */ +#include "zstd_bits.h" /* ZSTD_highbit32 */ /* ************************************************************** @@ -55,19 +54,6 @@ #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) - -/* Function templates */ -FSE_DTable* FSE_createDTable (unsigned tableLog) -{ - if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); -} - -void FSE_freeDTable (FSE_DTable* dt) -{ - ZSTD_free(dt); -} - static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) { void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ @@ -96,7 +82,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo symbolNext[s] = 1; } else { if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; - symbolNext[s] = normalizedCounter[s]; + symbolNext[s] = (U16)normalizedCounter[s]; } } } ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); } @@ -111,8 +97,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo * all symbols have counts <= 8. We ensure we have 8 bytes at the end of * our buffer to handle the over-write. */ - { - U64 const add = 0x0101010101010101ull; + { U64 const add = 0x0101010101010101ull; size_t pos = 0; U64 sv = 0; U32 s; @@ -123,14 +108,13 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo for (i = 8; i < n; i += 8) { MEM_write64(spread + pos + i, sv); } - pos += n; - } - } + pos += (size_t)n; + } } /* Now we spread those positions across the table. - * The benefit of doing it in two stages is that we avoid the the + * The benefit of doing it in two stages is that we avoid the * variable size inner loop, which caused lots of branch misses. * Now we can run through all the positions without any branch misses. - * We unroll the loop twice, since that is what emperically worked best. + * We unroll the loop twice, since that is what empirically worked best. */ { size_t position = 0; @@ -166,7 +150,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo for (u=0; u<tableSize; u++) { FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol); U32 const nextState = symbolNext[symbol]++; - tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) ); + tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) ); tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); } } @@ -184,49 +168,6 @@ size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsi /*-******************************************************* * Decompression (Byte symbols) *********************************************************/ -size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) -{ - void* ptr = dt; - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSE_decode_t* const cell = (FSE_decode_t*)dPtr; - - DTableH->tableLog = 0; - DTableH->fastMode = 0; - - cell->newState = 0; - cell->symbol = symbolValue; - cell->nbBits = 0; - - return 0; -} - - -size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) -{ - void* ptr = dt; - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSV1 = tableMask+1; - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* Build Decoding Table */ - DTableH->tableLog = (U16)nbBits; - DTableH->fastMode = 1; - for (s=0; s<maxSV1; s++) { - dinfo[s].newState = 0; - dinfo[s].symbol = (BYTE)s; - dinfo[s].nbBits = (BYTE)nbBits; - } - - return 0; -} FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic( void* dst, size_t maxDstSize, @@ -248,6 +189,8 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic( FSE_initDState(&state1, &bitD, dt); FSE_initDState(&state2, &bitD, dt); + RETURN_ERROR_IF(BIT_reloadDStream(&bitD)==BIT_DStream_overflow, corruption_detected, ""); + #define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) /* 4 symbols per loop */ @@ -287,32 +230,12 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic( break; } } - return op-ostart; -} - - -size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize, - const FSE_DTable* dt) -{ - const void* ptr = dt; - const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; - const U32 fastMode = DTableH->fastMode; - - /* select fast mode (static) */ - if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); - return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); -} - - -size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) -{ - return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0); + assert(op >= ostart); + return (size_t)(op-ostart); } typedef struct { short ncount[FSE_MAX_SYMBOL_VALUE + 1]; - FSE_DTable dtable[1]; /* Dynamically sized */ } FSE_DecompressWksp; @@ -327,13 +250,18 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( unsigned tableLog; unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace; + size_t const dtablePos = sizeof(FSE_DecompressWksp) / sizeof(FSE_DTable); + FSE_DTable* const dtable = (FSE_DTable*)workSpace + dtablePos; - DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0); + FSE_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0); if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC); + /* correct offset to dtable depends on this property */ + FSE_STATIC_ASSERT(sizeof(FSE_DecompressWksp) % sizeof(FSE_DTable) == 0); + /* normal FSE decoding mode */ - { - size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); + { size_t const NCountLength = + FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); if (FSE_isError(NCountLength)) return NCountLength; if (tableLog > maxLog) return ERROR(tableLog_tooLarge); assert(NCountLength <= cSrcSize); @@ -342,19 +270,20 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( } if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); - workSpace = wksp->dtable + FSE_DTABLE_SIZE_U32(tableLog); + assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize); + workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); - CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); + CHECK_F( FSE_buildDTable_internal(dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); { - const void* ptr = wksp->dtable; + const void* ptr = dtable; const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; const U32 fastMode = DTableH->fastMode; /* select fast mode (static) */ - if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1); - return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0); + if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, dtable, 1); + return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, dtable, 0); } } @@ -365,7 +294,7 @@ static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, co } #if DYNAMIC_BMI2 -TARGET_ATTRIBUTE("bmi2") static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) { return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); } @@ -382,9 +311,4 @@ size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); } - -typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; - - - #endif /* FSE_COMMONDEFS_ONLY */ diff --git a/src/bled/header_verbose_list.c b/src/bled/header_verbose_list.c index 73e1fee4c3a..a575a08a027 100644 --- a/src/bled/header_verbose_list.c +++ b/src/bled/header_verbose_list.c @@ -9,9 +9,9 @@ void FAST_FUNC header_verbose_list(const file_header_t *file_header) { struct tm tm_time; struct tm *ptm = &tm_time; //localtime(&file_header->mtime); + char modestr[12]; #if ENABLE_FEATURE_TAR_UNAME_GNAME - char modestr[12]; char uid[sizeof(int)*3 + 2]; /*char gid[sizeof(int)*3 + 2];*/ char *user; diff --git a/src/bled/huf.h b/src/bled/huf.h index 91e33c5417f..e4f5a6f45b2 100644 --- a/src/bled/huf.h +++ b/src/bled/huf.h @@ -1,7 +1,7 @@ /* ****************************************************************** * huff0 huffman codec, * part of Finite State Entropy library - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -12,91 +12,31 @@ * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ +#if defined (__cplusplus) +extern "C" { +#endif #ifndef HUF_H_298734234 #define HUF_H_298734234 /* *** Dependencies *** */ #include "zstd_deps.h" /* size_t */ - -# define HUF_PUBLIC_API - -/* ========================== */ -/* *** simple functions *** */ -/* ========================== */ - -/** HUF_compress() : - * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. - * 'dst' buffer must be already allocated. - * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). - * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. - * @return : size of compressed data (<= `dstCapacity`). - * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - * if HUF_isError(return), compression failed (more details using HUF_getErrorName()) - */ -HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/** HUF_decompress() : - * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', - * into already allocated buffer 'dst', of minimum size 'dstSize'. - * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. - * Note : in contrast with FSE, HUF_decompress can regenerate - * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, - * because it knows size to regenerate (originalSize). - * @return : size of regenerated data (== originalSize), - * or an error code, which can be tested using HUF_isError() - */ -HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize); +#include "zstd_mem.h" /* U32 */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" /* *** Tool functions *** */ -#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ -HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ /* Error Management */ -HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ -HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ - +unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ -/* *** Advanced function *** */ -/** HUF_compress2() : - * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`. - * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX . - * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ -HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned tableLog); - -/** HUF_compress4X_wksp() : - * Same as HUF_compress2(), but uses externally allocated `workSpace`. - * `workspace` must be at least as large as HUF_WORKSPACE_SIZE */ #define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */) #define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64)) -HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned tableLog, - void* workSpace, size_t wkspSize); - -#endif /* HUF_H_298734234 */ - -/* ****************************************************************** - * WARNING !! - * The following section contains advanced and experimental definitions - * which shall never be used in the context of a dynamic library, - * because they are not guaranteed to remain stable in the future. - * Only consider them in association with static linking. - * *****************************************************************/ -#if !defined(HUF_H_HUF_STATIC_LINKING_ONLY) -#define HUF_H_HUF_STATIC_LINKING_ONLY - -/* *** Dependencies *** */ -#include "zstd_mem.h" /* U32 */ -#define FSE_STATIC_LINKING_ONLY -#include "fse.h" - /* *** Constants *** */ #define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */ @@ -137,25 +77,49 @@ typedef U32 HUF_DTable; /* **************************************** * Advanced decompression functions ******************************************/ -HUF_PUBLIC_API size_t HUF_decompress4X1(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -HUF_PUBLIC_API size_t HUF_decompress4X2(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -#endif -HUF_PUBLIC_API size_t HUF_decompress4X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ -HUF_PUBLIC_API size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ -HUF_PUBLIC_API size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */ -HUF_PUBLIC_API size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -HUF_PUBLIC_API size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -HUF_PUBLIC_API size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -HUF_PUBLIC_API size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ -#endif +/** + * Huffman flags bitset. + * For all flags, 0 is the default value. + */ +typedef enum { + /** + * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime. + * Otherwise: Ignored. + */ + HUF_flags_bmi2 = (1 << 0), + /** + * If set: Test possible table depths to find the one that produces the smallest header + encoded size. + * If unset: Use heuristic to find the table depth. + */ + HUF_flags_optimalDepth = (1 << 1), + /** + * If set: If the previous table can encode the input, always reuse the previous table. + * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output. + */ + HUF_flags_preferRepeat = (1 << 2), + /** + * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress. + * If unset: Always histogram the entire input. + */ + HUF_flags_suspectUncompressible = (1 << 3), + /** + * If set: Don't use assembly implementations + * If unset: Allow using assembly implementations + */ + HUF_flags_disableAsm = (1 << 4), + /** + * If set: Don't use the fast decoding loop, always use the fallback decoding loop. + * If unset: Use the fast decoding loop when possible. + */ + HUF_flags_disableFast = (1 << 5) +} HUF_flags_e; /* **************************************** * HUF detailed API * ****************************************/ +#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra /*! HUF_compress() does the following: * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") @@ -168,39 +132,40 @@ HUF_PUBLIC_API size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, s * For example, it's possible to compress several blocks using the same 'CTable', * or to save and regenerate 'CTable' using external methods. */ -HUF_PUBLIC_API unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); -HUF_PUBLIC_API size_t HUF_buildCTable(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */ -HUF_PUBLIC_API size_t HUF_writeCTable(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); -HUF_PUBLIC_API size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); -HUF_PUBLIC_API size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); -HUF_PUBLIC_API size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2); -HUF_PUBLIC_API size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); -HUF_PUBLIC_API int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); +unsigned HUF_minTableLog(unsigned symbolCardinality); +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue); +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, + size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */ +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); typedef enum { HUF_repeat_none, /**< Cannot use the previous table */ HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ } HUF_repeat; + /** HUF_compress4X_repeat() : * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * If it uses hufTable it does not modify hufTable or repeat. * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. * If preferRepeat then the old table will always be used if valid. * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ -HUF_PUBLIC_API size_t HUF_compress4X_repeat(void* dst, size_t dstSize, +size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible); + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); /** HUF_buildCTable_wksp() : * Same as HUF_buildCTable(), but using externally allocated scratch buffer. * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. */ -#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1) +#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192) #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) -HUF_PUBLIC_API size_t HUF_buildCTable_wksp(HUF_CElt* tree, +size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize); @@ -209,7 +174,7 @@ HUF_PUBLIC_API size_t HUF_buildCTable_wksp(HUF_CElt* tree, * `huffWeight` is destination buffer. * @return : size read from `src` , or an error Code . * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ -HUF_PUBLIC_API size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize); @@ -220,20 +185,33 @@ HUF_PUBLIC_API size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, */ #define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1) #define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned)) -HUF_PUBLIC_API size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workspace, size_t wkspSize, - int bmi2); + int flags); /** HUF_readCTable() : * Loading a CTable saved with HUF_writeCTable() */ -HUF_PUBLIC_API size_t HUF_readCTable(HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); /** HUF_getNbBitsFromCTable() : * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX - * Note 1 : is not inlined, as HUF_CElt definition is private */ -HUF_PUBLIC_API U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue); + * Note 1 : If symbolValue > HUF_readCTableHeader(symbolTable).maxSymbolValue, returns 0 + * Note 2 : is not inlined, as HUF_CElt definition is private + */ +U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue); + +typedef struct { + BYTE tableLog; + BYTE maxSymbolValue; + BYTE unused[sizeof(size_t) - 2]; +} HUF_CTableHeader; + +/** HUF_readCTableHeader() : + * @returns The header from the CTable specifying the tableLog and the maxSymbolValue. + */ +HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable); /* * HUF_decompress() does the following: @@ -247,7 +225,7 @@ HUF_PUBLIC_API U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symb * based on a set of pre-computed metrics. * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . * Assumption : 0 < dstSize <= 128 KB */ -HUF_PUBLIC_API U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize); +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); /** * The minimum workspace size for the `workSpace` used in @@ -262,83 +240,47 @@ HUF_PUBLIC_API U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize); #define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9)) #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) -#ifndef HUF_FORCE_DECOMPRESS_X2 -HUF_PUBLIC_API size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize); -HUF_PUBLIC_API size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -HUF_PUBLIC_API size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize); -HUF_PUBLIC_API size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); -#endif - -HUF_PUBLIC_API size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#ifndef HUF_FORCE_DECOMPRESS_X2 -HUF_PUBLIC_API size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -HUF_PUBLIC_API size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif - /* ====================== */ /* single stream variants */ /* ====================== */ -HUF_PUBLIC_API size_t HUF_compress1X(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); -HUF_PUBLIC_API size_t HUF_compress1X_wksp(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U64 U64 */ -HUF_PUBLIC_API size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); -HUF_PUBLIC_API size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2); +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); /** HUF_compress1X_repeat() : * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * If it uses hufTable it does not modify hufTable or repeat. * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. * If preferRepeat then the old table will always be used if valid. * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ -HUF_PUBLIC_API size_t HUF_compress1X_repeat(void* dst, size_t dstSize, +size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible); + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); -HUF_PUBLIC_API size_t HUF_decompress1X1(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #ifndef HUF_FORCE_DECOMPRESS_X1 -HUF_PUBLIC_API size_t HUF_decompress1X2(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ -#endif - -HUF_PUBLIC_API size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); -HUF_PUBLIC_API size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); -#ifndef HUF_FORCE_DECOMPRESS_X2 -HUF_PUBLIC_API size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -HUF_PUBLIC_API size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -HUF_PUBLIC_API size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -HUF_PUBLIC_API size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ -#endif - -HUF_PUBLIC_API size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ -#ifndef HUF_FORCE_DECOMPRESS_X2 -HUF_PUBLIC_API size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -HUF_PUBLIC_API size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */ #endif /* BMI2 variants. * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. */ -size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); #ifndef HUF_FORCE_DECOMPRESS_X2 -HUF_PUBLIC_API size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #endif -HUF_PUBLIC_API size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); -size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #ifndef HUF_FORCE_DECOMPRESS_X2 -HUF_PUBLIC_API size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); #endif #ifndef HUF_FORCE_DECOMPRESS_X1 -HUF_PUBLIC_API size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); #endif -#endif /* HUF_STATIC_LINKING_ONLY */ +#endif /* HUF_H_298734234 */ +#if defined (__cplusplus) +} +#endif diff --git a/src/bled/huf_decompress.c b/src/bled/huf_decompress.c index 5a019c7d112..d870f112eee 100644 --- a/src/bled/huf_decompress.c +++ b/src/bled/huf_decompress.c @@ -1,7 +1,7 @@ /* ****************************************************************** * huff0 huffman decoder, * part of Finite State Entropy library - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -19,10 +19,10 @@ #include "zstd_compiler.h" #include "fse_bitstream.h" /* BIT_* */ #include "fse.h" /* to compress headers */ -#define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "zstd_error_private.h" #include "zstd_internal.h" +#include "zstd_bits.h" /* ZSTD_highbit32, ZSTD_countTrailingZeros64 */ /* ************************************************************** * Constants @@ -34,6 +34,12 @@ * Macros ****************************************************************/ +#ifdef HUF_DISABLE_FAST_DECODE +# define HUF_ENABLE_FAST_DECODE 0 +#else +# define HUF_ENABLE_FAST_DECODE 1 +#endif + /* These two optional macros force the use one way or another of the two * Huffman decompression implementations. You can't force in both directions * at the same time. @@ -43,32 +49,29 @@ #error "Cannot force the use of the X1 and X2 decoders at the same time!" #endif -# define HUF_ASM_SUPPORTED 0 - -/* HUF_DISABLE_ASM: Disables all ASM implementations. */ -#define HUF_ENABLE_ASM_X86_64_BMI2 0 - -#if HUF_ENABLE_ASM_X86_64_BMI2 && DYNAMIC_BMI2 -# define HUF_ASM_X86_64_BMI2_ATTRS TARGET_ATTRIBUTE("bmi2") +/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is + * supported at runtime, so we can add the BMI2 target attribute. + * When it is disabled, we will still get BMI2 if it is enabled statically. + */ +#if DYNAMIC_BMI2 +# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE #else -# define HUF_ASM_X86_64_BMI2_ATTRS +# define HUF_FAST_BMI2_ATTRS #endif -#define HUF_EXTERN_C +#ifdef __cplusplus +# define HUF_EXTERN_C extern "C" +#else +# define HUF_EXTERN_C +#endif #define HUF_ASM_DECL HUF_EXTERN_C -#if DYNAMIC_BMI2 || (HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)) +#if DYNAMIC_BMI2 # define HUF_NEED_BMI2_FUNCTION 1 #else # define HUF_NEED_BMI2_FUNCTION 0 #endif -#if !(HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)) -# define HUF_NEED_DEFAULT_FUNCTION 1 -#else -# define HUF_NEED_DEFAULT_FUNCTION 0 -#endif - /* ************************************************************** * Error Management ****************************************************************/ @@ -85,6 +88,11 @@ /* ************************************************************** * BMI2 Variant Wrappers ****************************************************************/ +typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize, + const void *cSrc, + size_t cSrcSize, + const HUF_DTable *DTable); + #if DYNAMIC_BMI2 #define HUF_DGEN(fn) \ @@ -97,7 +105,7 @@ return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ } \ \ - static TARGET_ATTRIBUTE("bmi2") size_t fn##_bmi2( \ + static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \ void* dst, size_t dstSize, \ const void* cSrc, size_t cSrcSize, \ const HUF_DTable* DTable) \ @@ -106,9 +114,9 @@ } \ \ static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ { \ - if (bmi2) { \ + if (flags & HUF_flags_bmi2) { \ return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ } \ return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ @@ -118,9 +126,9 @@ #define HUF_DGEN(fn) \ static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ { \ - (void)bmi2; \ + (void)flags; \ return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ } @@ -139,42 +147,66 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) return dtd; } -#if HUF_ENABLE_ASM_X86_64_BMI2 - -static size_t HUF_initDStream(BYTE const* ip) { +static size_t HUF_initFastDStream(BYTE const* ip) { BYTE const lastByte = ip[7]; - size_t const bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; size_t const value = MEM_readLEST(ip) | 1; assert(bitsConsumed <= 8); + assert(sizeof(size_t) == 8); return value << bitsConsumed; } + + +/** + * The input/output arguments to the Huffman fast decoding loop: + * + * ip [in/out] - The input pointers, must be updated to reflect what is consumed. + * op [in/out] - The output pointers, must be updated to reflect what is written. + * bits [in/out] - The bitstream containers, must be updated to reflect the current state. + * dt [in] - The decoding table. + * ilowest [in] - The beginning of the valid range of the input. Decoders may read + * down to this pointer. It may be below iend[0]. + * oend [in] - The end of the output stream. op[3] must not cross oend. + * iend [in] - The end of each input stream. ip[i] may cross iend[i], + * as long as it is above ilowest, but that indicates corruption. + */ typedef struct { BYTE const* ip[4]; BYTE* op[4]; U64 bits[4]; void const* dt; - BYTE const* ilimit; + BYTE const* ilowest; BYTE* oend; BYTE const* iend[4]; -} HUF_DecompressAsmArgs; +} HUF_DecompressFastArgs; + +typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*); /** - * Initializes args for the asm decoding loop. - * @returns 0 on success - * 1 if the fallback implementation should be used. + * Initializes args for the fast decoding loop. + * @returns 1 on success + * 0 if the fallback implementation should be used. * Or an error code on failure. */ -static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) +static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) { void const* dt = DTable + 1; U32 const dtLog = HUF_getDTableDesc(DTable).tableLog; - const BYTE* const ilimit = (const BYTE*)src + 6 + 8; + const BYTE* const istart = (const BYTE*)src; + + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); - BYTE* const oend = (BYTE*)dst + dstSize; + /* The fast decoding loop assumes 64-bit little-endian. + * This condition is false on x32. + */ + if (!MEM_isLittleEndian() || MEM_32bits()) + return 0; - /* We're assuming x86-64 BMI2 - assure that this is the case. */ - assert(MEM_isLittleEndian() && !MEM_32bits()); + /* Avoid nullptr addition */ + if (dstSize == 0) + return 0; + assert(dst != NULL); /* strict minimum : jump table + 1 byte per stream */ if (srcSize < 10) @@ -185,11 +217,10 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder. */ if (dtLog != HUF_DECODER_FAST_TABLELOG) - return 1; + return 0; /* Read the jump table. */ { - const BYTE* const istart = (const BYTE*)src; size_t const length1 = MEM_readLE16(istart); size_t const length2 = MEM_readLE16(istart+2); size_t const length3 = MEM_readLE16(istart+4); @@ -199,13 +230,11 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, args->iend[2] = args->iend[1] + length2; args->iend[3] = args->iend[2] + length3; - /* HUF_initDStream() requires this, and this small of an input + /* HUF_initFastDStream() requires this, and this small of an input * won't benefit from the ASM loop anyways. - * length1 must be >= 16 so that ip[0] >= ilimit before the loop - * starts. */ - if (length1 < 16 || length2 < 8 || length3 < 8 || length4 < 8) - return 1; + if (length1 < 8 || length2 < 8 || length3 < 8 || length4 < 8) + return 0; if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */ } /* ip[] contains the position that is currently loaded into bits[]. */ @@ -222,7 +251,7 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, /* No point to call the ASM loop for tiny outputs. */ if (args->op[3] >= oend) - return 1; + return 0; /* bits[] is the bit container. * It is read from the MSB down to the LSB. @@ -231,24 +260,25 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, * set, so that CountTrailingZeros(bits[]) can be used * to count how many bits we've consumed. */ - args->bits[0] = HUF_initDStream(args->ip[0]); - args->bits[1] = HUF_initDStream(args->ip[1]); - args->bits[2] = HUF_initDStream(args->ip[2]); - args->bits[3] = HUF_initDStream(args->ip[3]); - - /* If ip[] >= ilimit, it is guaranteed to be safe to - * reload bits[]. It may be beyond its section, but is - * guaranteed to be valid (>= istart). - */ - args->ilimit = ilimit; + args->bits[0] = HUF_initFastDStream(args->ip[0]); + args->bits[1] = HUF_initFastDStream(args->ip[1]); + args->bits[2] = HUF_initFastDStream(args->ip[2]); + args->bits[3] = HUF_initFastDStream(args->ip[3]); + + /* The decoders must be sure to never read beyond ilowest. + * This is lower than iend[0], but allowing decoders to read + * down to ilowest can allow an extra iteration or two in the + * fast loop. + */ + args->ilowest = istart; args->oend = oend; args->dt = dt; - return 0; + return 1; } -static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs const* args, int stream, BYTE* segmentEnd) +static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd) { /* Validate that we haven't overwritten. */ if (args->op[stream] > segmentEnd) @@ -262,15 +292,33 @@ static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs return ERROR(corruption_detected); /* Construct the BIT_DStream_t. */ - bit->bitContainer = MEM_readLE64(args->ip[stream]); - bit->bitsConsumed = ZSTD_countTrailingZeros((size_t)args->bits[stream]); - bit->start = (const char*)args->iend[0]; + assert(sizeof(size_t) == 8); + bit->bitContainer = MEM_readLEST(args->ip[stream]); + bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]); + bit->start = (const char*)args->ilowest; bit->limitPtr = bit->start + sizeof(size_t); bit->ptr = (const char*)args->ip[stream]; return 0; } -#endif + +/* Calls X(N) for each stream 0, 1, 2, 3. */ +#define HUF_4X_FOR_EACH_STREAM(X) \ + do { \ + X(0); \ + X(1); \ + X(2); \ + X(3); \ + } while (0) + +/* Calls X(N, var) for each stream 0, 1, 2, 3. */ +#define HUF_4X_FOR_EACH_STREAM_WITH_VAR(X, var) \ + do { \ + X(0, (var)); \ + X(1, (var)); \ + X(2, (var)); \ + X(3, (var)); \ + } while (0) #ifndef HUF_FORCE_DECOMPRESS_X2 @@ -287,10 +335,11 @@ typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decodi static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { U64 D4; if (MEM_isLittleEndian()) { - D4 = (symbol << 8) + nbBits; + D4 = (U64)((symbol << 8) + nbBits); } else { - D4 = symbol + (nbBits << 8); + D4 = (U64)(symbol + (nbBits << 8)); } + assert(D4 < (1U << 16)); D4 *= 0x0001000100010001ULL; return D4; } @@ -333,13 +382,7 @@ typedef struct { BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; } HUF_ReadDTableX1_Workspace; - -size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) -{ - return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0); -} - -size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2) +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags) { U32 tableLog = 0; U32 nbSymbols = 0; @@ -354,7 +397,7 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ - iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2); + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags); if (HUF_isError(iSize)) return iSize; @@ -381,9 +424,8 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr * rankStart[0] is not filled because there are no entries in the table for * weight 0. */ - { - int n; - int nextRankStart = 0; + { int n; + U32 nextRankStart = 0; int const unroll = 4; int const nLimit = (int)nbSymbols - unroll + 1; for (n=0; n<(int)tableLog+1; n++) { @@ -406,14 +448,13 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr /* fill DTable * We fill all entries of each weight in order. - * That way length is a constant for each iteration of the outter loop. + * That way length is a constant for each iteration of the outer loop. * We can switch based on the length to a different inner loop which is * optimized for that particular case. */ - { - U32 w; - int symbol=wksp->rankVal[0]; - int rankStart=0; + { U32 w; + int symbol = wksp->rankVal[0]; + int rankStart = 0; for (w=1; w<tableLog+1; ++w) { int const symbolCount = wksp->rankVal[w]; int const length = (1 << w) >> 1; @@ -487,15 +528,19 @@ HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog } #define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ - *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog) + do { *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog); } while (0) -#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ - HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) +#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \ + } while (0) -#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) +#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \ + } while (0) HINT_INLINE size_t HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) @@ -510,6 +555,8 @@ HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, cons HUF_DECODE_SYMBOLX1_2(p, bitDPtr); HUF_DECODE_SYMBOLX1_0(p, bitDPtr); } + } else { + BIT_reloadDStream(bitDPtr); } /* [0-3] symbols remaining */ @@ -521,7 +568,7 @@ HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, cons while (p < pEnd) HUF_DECODE_SYMBOLX1_0(p, bitDPtr); - return pEnd-pStart; + return (size_t)(pEnd-pStart); } FORCE_INLINE_TEMPLATE size_t @@ -531,7 +578,7 @@ HUF_decompress1X1_usingDTable_internal_body( const HUF_DTable* DTable) { BYTE* op = (BYTE*)dst; - BYTE* const oend = op + dstSize; + BYTE* const oend = ZSTD_maybeNullPtrAdd(op, dstSize); const void* dtPtr = DTable + 1; const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; BIT_DStream_t bitD; @@ -547,6 +594,10 @@ HUF_decompress1X1_usingDTable_internal_body( return dstSize; } +/* HUF_decompress4X1_usingDTable_internal_body(): + * Conditions : + * @dstSize >= 6 + */ FORCE_INLINE_TEMPLATE size_t HUF_decompress4X1_usingDTable_internal_body( void* dst, size_t dstSize, @@ -555,6 +606,7 @@ HUF_decompress4X1_usingDTable_internal_body( { /* Check */ if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ { const BYTE* const istart = (const BYTE*) cSrc; BYTE* const ostart = (BYTE*) dst; @@ -590,6 +642,7 @@ HUF_decompress4X1_usingDTable_internal_body( if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + assert(dstSize >= 6); /* validated above */ CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); @@ -645,59 +698,180 @@ HUF_decompress4X1_usingDTable_internal_body( } #if HUF_NEED_BMI2_FUNCTION -static TARGET_ATTRIBUTE("bmi2") +static BMI2_TARGET_ATTRIBUTE size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, size_t cSrcSize, HUF_DTable const* DTable) { return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); } #endif -#if HUF_NEED_DEFAULT_FUNCTION static size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, size_t cSrcSize, HUF_DTable const* DTable) { return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); } + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + #endif -#if HUF_ENABLE_ASM_X86_64_BMI2 +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + U16 const* const dtable = (U16 const*)args->dt; + BYTE* const oend = args->oend; + BYTE const* const ilowest = args->ilowest; + + /* Copy the arguments to local variables */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= (stream == 3 ? oend : op[stream + 1])); + assert(ip[stream] >= ilowest); + } +#endif + /* Compute olimit */ + { + /* Each iteration produces 5 output symbols per stream */ + size_t const oiters = (size_t)(oend - op[3]) / 5; + /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes + * per stream. + */ + size_t const iiters = (size_t)(ip[0] - ilowest) / 7; + /* We can safely run iters iterations before running bounds checks */ + size_t const iters = MIN(oiters, iiters); + size_t const symbols = iters * 5; + + /* We can simply check that op[3] < olimit, instead of checking all + * of our bounds, since we can't hit the other bounds until we've run + * iters iterations, which only happens when op[3] == olimit. + */ + olimit = op[3] + symbols; + + /* Exit fast decoding loop once we reach the end. */ + if (op[3] == olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } -HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args); +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif -static HUF_ASM_X86_64_BMI2_ATTRS +#define HUF_4X1_DECODE_SYMBOL(_stream, _symbol) \ + do { \ + int const index = (int)(bits[(_stream)] >> 53); \ + int const entry = (int)dtable[index]; \ + bits[(_stream)] <<= (entry & 0x3F); \ + op[(_stream)][(_symbol)] = (BYTE)((entry >> 8) & 0xFF); \ + } while (0) + +#define HUF_4X1_RELOAD_STREAM(_stream) \ + do { \ + int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ + int const nbBits = ctz & 7; \ + int const nbBytes = ctz >> 3; \ + op[(_stream)] += 5; \ + ip[(_stream)] -= nbBytes; \ + bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ + bits[(_stream)] <<= nbBits; \ + } while (0) + + /* Manually unroll the loop because compilers don't consistently + * unroll the inner loops, which destroys performance. + */ + do { + /* Decode 5 symbols in each of the 4 streams */ + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 1); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 2); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 3); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 4); + + /* Reload each of the 4 the bitstreams */ + HUF_4X_FOR_EACH_STREAM(HUF_4X1_RELOAD_STREAM); + } while (op[3] < olimit); + +#undef HUF_4X1_DECODE_SYMBOL +#undef HUF_4X1_RELOAD_STREAM + } + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + +/** + * @returns @p dstSize on success (>= 6) + * 0 if the fallback implementation should be used + * An error if an error occurred + */ +static HUF_FAST_BMI2_ATTRS size_t -HUF_decompress4X1_usingDTable_internal_bmi2_asm( +HUF_decompress4X1_usingDTable_internal_fast( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) { void const* dt = DTable + 1; - const BYTE* const iend = (const BYTE*)cSrc + 6; - BYTE* const oend = (BYTE*)dst + dstSize; - HUF_DecompressAsmArgs args; - { - size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); - FORWARD_IF_ERROR(ret, "Failed to init asm args"); - if (ret != 0) - return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); + BYTE const* const ilowest = (BYTE const*)cSrc; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + HUF_DecompressFastArgs args; + { size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init fast loop args"); + if (ret == 0) + return 0; } - assert(args.ip[0] >= args.ilimit); - HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(&args); + assert(args.ip[0] >= args.ilowest); + loopFn(&args); - /* Our loop guarantees that ip[] >= ilimit and that we haven't + /* Our loop guarantees that ip[] >= ilowest and that we haven't * overwritten any op[]. */ - assert(args.ip[0] >= iend); - assert(args.ip[1] >= iend); - assert(args.ip[2] >= iend); - assert(args.ip[3] >= iend); + assert(args.ip[0] >= ilowest); + assert(args.ip[0] >= ilowest); + assert(args.ip[1] >= ilowest); + assert(args.ip[2] >= ilowest); + assert(args.ip[3] >= ilowest); assert(args.op[3] <= oend); - (void)iend; + + assert(ilowest == args.ilowest); + assert(ilowest + 6 == args.iend[0]); + (void)ilowest; /* finish bit streams one by one. */ - { - size_t const segmentSize = (dstSize+3) / 4; + { size_t const segmentSize = (dstSize+3) / 4; BYTE* segmentEnd = (BYTE*)dst; int i; for (i = 0; i < 4; ++i) { @@ -714,97 +888,59 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm( } /* decoded size */ + assert(dstSize != 0); return dstSize; } -#endif /* HUF_ENABLE_ASM_X86_64_BMI2 */ - -typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize, - const void *cSrc, - size_t cSrcSize, - const HUF_DTable *DTable); HUF_DGEN(HUF_decompress1X1_usingDTable_internal) static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) + size_t cSrcSize, HUF_DTable const* DTable, int flags) { + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop; + #if DYNAMIC_BMI2 - if (bmi2) { -# if HUF_ENABLE_ASM_X86_64_BMI2 - return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); -# else - return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } # endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); } -#else - (void)bmi2; #endif -#if HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) - return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); -#else - return HUF_decompress4X1_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable); +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } #endif -} - - -size_t HUF_decompress1X1_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} -size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); -} - - -size_t HUF_decompress4X1_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); + if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); } -static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, +static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize, int bmi2) + void* workSpace, size_t wkspSize, int flags) { const BYTE* ip = (const BYTE*) cSrc; - size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; - return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); -} - -size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0); + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); } - #endif /* HUF_FORCE_DECOMPRESS_X2 */ @@ -987,7 +1123,7 @@ static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, const sortedSymbol_t* sortedList, - const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight, const U32 nbBitsBaseline) { U32* const rankVal = rankValOrigin[0]; @@ -1042,14 +1178,7 @@ typedef struct { size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, - void* workSpace, size_t wkspSize) -{ - return HUF_readDTableX2_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0); -} - -size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, - const void* src, size_t srcSize, - void* workSpace, size_t wkspSize, int bmi2) + void* workSpace, size_t wkspSize, int flags) { U32 tableLog, maxW, nbSymbols; DTableDesc dtd = HUF_getDTableDesc(DTable); @@ -1071,7 +1200,7 @@ size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ - iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), bmi2); + iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags); if (HUF_isError(iSize)) return iSize; /* check result */ @@ -1121,17 +1250,10 @@ size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, rankValPtr[w] = rankVal0[w] >> consumed; } } } } -#if defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wstringop-overflow" -#endif HUF_fillDTableX2(dt, maxTableLog, wksp->sortedSymbol, wksp->rankStart0, wksp->rankVal, maxW, tableLog+1); -#if defined(__GNUC__) -#pragma GCC diagnostic pop -#endif dtd.tableLog = (BYTE)maxTableLog; dtd.tableType = 1; @@ -1168,15 +1290,19 @@ HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, c } #define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + do { ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); } while (0) -#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \ + } while (0) -#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \ + } while (0) HINT_INLINE size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, @@ -1204,6 +1330,8 @@ HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, HUF_DECODE_SYMBOLX2_0(p, bitDPtr); } } + } else { + BIT_reloadDStream(bitDPtr); } /* closer to end : up to 2 symbols at a time */ @@ -1234,7 +1362,7 @@ HUF_decompress1X2_usingDTable_internal_body( /* decode */ { BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; + BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, dstSize); const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; DTableDesc const dtd = HUF_getDTableDesc(DTable); @@ -1247,6 +1375,11 @@ HUF_decompress1X2_usingDTable_internal_body( /* decoded size */ return dstSize; } + +/* HUF_decompress4X2_usingDTable_internal_body(): + * Conditions: + * @dstSize >= 6 + */ FORCE_INLINE_TEMPLATE size_t HUF_decompress4X2_usingDTable_internal_body( void* dst, size_t dstSize, @@ -1254,6 +1387,7 @@ HUF_decompress4X2_usingDTable_internal_body( const HUF_DTable* DTable) { if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ { const BYTE* const istart = (const BYTE*) cSrc; BYTE* const ostart = (BYTE*) dst; @@ -1287,8 +1421,9 @@ HUF_decompress4X2_usingDTable_internal_body( DTableDesc const dtd = HUF_getDTableDesc(DTable); U32 const dtLog = dtd.tableLog; - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + assert(dstSize >= 6 /* validated above */); CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); @@ -1335,7 +1470,7 @@ HUF_decompress4X2_usingDTable_internal_body( HUF_DECODE_SYMBOLX2_0(op2, &bitD2); HUF_DECODE_SYMBOLX2_0(op3, &bitD3); HUF_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal = (U32)LIKELY( + endSignal = (U32)LIKELY((U32) (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) @@ -1366,51 +1501,198 @@ HUF_decompress4X2_usingDTable_internal_body( } #if HUF_NEED_BMI2_FUNCTION -static TARGET_ATTRIBUTE("bmi2") +static BMI2_TARGET_ATTRIBUTE size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, size_t cSrcSize, HUF_DTable const* DTable) { return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); } #endif -#if HUF_NEED_DEFAULT_FUNCTION static size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, size_t cSrcSize, HUF_DTable const* DTable) { return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); } + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + #endif -#if HUF_ENABLE_ASM_X86_64_BMI2 +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + BYTE* oend[4]; + HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt; + BYTE const* const ilowest = args->ilowest; + + /* Copy the arguments to local registers. */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + oend[0] = op[1]; + oend[1] = op[2]; + oend[2] = op[3]; + oend[3] = args->oend; + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= oend[stream]); + assert(ip[stream] >= ilowest); + } +#endif + /* Compute olimit */ + { + /* Each loop does 5 table lookups for each of the 4 streams. + * Each table lookup consumes up to 11 bits of input, and produces + * up to 2 bytes of output. + */ + /* We can consume up to 7 bytes of input per iteration per stream. + * We also know that each input pointer is >= ip[0]. So we can run + * iters loops before running out of input. + */ + size_t iters = (size_t)(ip[0] - ilowest) / 7; + /* Each iteration can produce up to 10 bytes of output per stream. + * Each output stream my advance at different rates. So take the + * minimum number of safe iterations among all the output streams. + */ + for (stream = 0; stream < 4; ++stream) { + size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10; + iters = MIN(iters, oiters); + } + + /* Each iteration produces at least 5 output symbols. So until + * op[3] crosses olimit, we know we haven't executed iters + * iterations yet. This saves us maintaining an iters counter, + * at the expense of computing the remaining # of iterations + * more frequently. + */ + olimit = op[3] + (iters * 5); -HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args); + /* Exit the fast decoding loop once we reach the end. */ + if (op[3] == olimit) + break; -static HUF_ASM_X86_64_BMI2_ATTRS size_t -HUF_decompress4X2_usingDTable_internal_bmi2_asm( + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + +#define HUF_4X2_DECODE_SYMBOL(_stream, _decode3) \ + do { \ + if ((_decode3) || (_stream) != 3) { \ + int const index = (int)(bits[(_stream)] >> 53); \ + HUF_DEltX2 const entry = dtable[index]; \ + MEM_write16(op[(_stream)], entry.sequence); \ + bits[(_stream)] <<= (entry.nbBits) & 0x3F; \ + op[(_stream)] += (entry.length); \ + } \ + } while (0) + +#define HUF_4X2_RELOAD_STREAM(_stream) \ + do { \ + HUF_4X2_DECODE_SYMBOL(3, 1); \ + { \ + int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ + int const nbBits = ctz & 7; \ + int const nbBytes = ctz >> 3; \ + ip[(_stream)] -= nbBytes; \ + bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ + bits[(_stream)] <<= nbBits; \ + } \ + } while (0) + + /* Manually unroll the loop because compilers don't consistently + * unroll the inner loops, which destroys performance. + */ + do { + /* Decode 5 symbols from each of the first 3 streams. + * The final stream will be decoded during the reload phase + * to reduce register pressure. + */ + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + + /* Decode one symbol from the final stream */ + HUF_4X2_DECODE_SYMBOL(3, 1); + + /* Decode 4 symbols from the final stream & reload bitstreams. + * The final stream is reloaded last, meaning that all 5 symbols + * are decoded from the final stream before it is reloaded. + */ + HUF_4X_FOR_EACH_STREAM(HUF_4X2_RELOAD_STREAM); + } while (op[3] < olimit); + } + +#undef HUF_4X2_DECODE_SYMBOL +#undef HUF_4X2_RELOAD_STREAM + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + + +static HUF_FAST_BMI2_ATTRS size_t +HUF_decompress4X2_usingDTable_internal_fast( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) { + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) { void const* dt = DTable + 1; - const BYTE* const iend = (const BYTE*)cSrc + 6; - BYTE* const oend = (BYTE*)dst + dstSize; - HUF_DecompressAsmArgs args; + const BYTE* const ilowest = (const BYTE*)cSrc; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + HUF_DecompressFastArgs args; { - size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); FORWARD_IF_ERROR(ret, "Failed to init asm args"); - if (ret != 0) - return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); + if (ret == 0) + return 0; } - assert(args.ip[0] >= args.ilimit); - HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(&args); + assert(args.ip[0] >= args.ilowest); + loopFn(&args); /* note : op4 already verified within main loop */ - assert(args.ip[0] >= iend); - assert(args.ip[1] >= iend); - assert(args.ip[2] >= iend); - assert(args.ip[3] >= iend); + assert(args.ip[0] >= ilowest); + assert(args.ip[1] >= ilowest); + assert(args.ip[2] >= ilowest); + assert(args.ip[3] >= ilowest); assert(args.op[3] <= oend); - (void)iend; + + assert(ilowest == args.ilowest); + assert(ilowest + 6 == args.iend[0]); + (void)ilowest; /* finish bitStreams one by one */ { @@ -1433,91 +1715,72 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm( /* decoded size */ return dstSize; } -#endif /* HUF_ENABLE_ASM_X86_64_BMI2 */ static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) + size_t cSrcSize, HUF_DTable const* DTable, int flags) { + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop; + #if DYNAMIC_BMI2 - if (bmi2) { -# if HUF_ENABLE_ASM_X86_64_BMI2 - return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); -# else - return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } # endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); } -#else - (void)bmi2; #endif -#if HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) - return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); -#else - return HUF_decompress4X2_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable); +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } #endif + + if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); } HUF_DGEN(HUF_decompress1X2_usingDTable_internal) -size_t HUF_decompress1X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} - size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) + void* workSpace, size_t wkspSize, int flags) { const BYTE* ip = (const BYTE*) cSrc; size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, - workSpace, wkspSize); + workSpace, wkspSize, flags); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags); } - -size_t HUF_decompress4X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} - -static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, +static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize, int bmi2) + void* workSpace, size_t wkspSize, int flags) { const BYTE* ip = (const BYTE*) cSrc; size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, - workSpace, wkspSize); + workSpace, wkspSize, flags); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); } -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0); -} - - #endif /* HUF_FORCE_DECOMPRESS_X1 */ @@ -1525,44 +1788,6 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, /* Universal decompression selectors */ /* ***********************************/ -size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#else - return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : - HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#endif -} - -size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#else - return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : - HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#endif -} - #if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; @@ -1617,36 +1842,9 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) #endif } - -size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, - size_t dstSize, const void* cSrc, - size_t cSrcSize, void* workSpace, - size_t wkspSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize == 0) return ERROR(corruption_detected); - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#else - return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize): - HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#endif - } -} - size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) + void* workSpace, size_t wkspSize, int flags) { /* validation checks */ if (dstSize == 0) return ERROR(dstSize_tooSmall); @@ -1659,71 +1857,71 @@ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, (void)algoNb; assert(algoNb == 0); return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); + cSrcSize, workSpace, wkspSize, flags); #elif defined(HUF_FORCE_DECOMPRESS_X2) (void)algoNb; assert(algoNb == 1); return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); + cSrcSize, workSpace, wkspSize, flags); #else return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize): + cSrcSize, workSpace, wkspSize, flags): HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); + cSrcSize, workSpace, wkspSize, flags); #endif } } -size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) { DTableDesc const dtd = HUF_getDTableDesc(DTable); #if defined(HUF_FORCE_DECOMPRESS_X1) (void)dtd; assert(dtd.tableType == 0); - return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #elif defined(HUF_FORCE_DECOMPRESS_X2) (void)dtd; assert(dtd.tableType == 1); - return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #else - return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : - HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #endif } #ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) { const BYTE* ip = (const BYTE*) cSrc; - size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); } #endif -size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) { DTableDesc const dtd = HUF_getDTableDesc(DTable); #if defined(HUF_FORCE_DECOMPRESS_X1) (void)dtd; assert(dtd.tableType == 0); - return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #elif defined(HUF_FORCE_DECOMPRESS_X2) (void)dtd; assert(dtd.tableType == 1); - return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #else - return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : - HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #endif } -size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) { /* validation checks */ if (dstSize == 0) return ERROR(dstSize_tooSmall); @@ -1733,15 +1931,14 @@ size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t ds #if defined(HUF_FORCE_DECOMPRESS_X1) (void)algoNb; assert(algoNb == 0); - return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); #elif defined(HUF_FORCE_DECOMPRESS_X2) (void)algoNb; assert(algoNb == 1); - return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); #else - return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) : - HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) : + HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); #endif } } - diff --git a/src/bled/init_handle.c b/src/bled/init_handle.c index c9c3e5118b8..b2d1d984d7e 100644 --- a/src/bled/init_handle.c +++ b/src/bled/init_handle.c @@ -2,7 +2,6 @@ /* * Licensed under GPLv2 or later, see file LICENSE in this source tree. */ - #include "libbb.h" #include "bb_archive.h" diff --git a/src/bled/libbb.h b/src/bled/libbb.h index 7d17823663e..94ae9dd4b63 100644 --- a/src/bled/libbb.h +++ b/src/bled/libbb.h @@ -39,24 +39,37 @@ #include <sys/types.h> #include <io.h> -#define ONE_TB 1099511627776ULL +#define ONE_TB 1099511627776ULL -#define ENABLE_DESKTOP 1 +#define ENABLE_DESKTOP 1 #if ENABLE_DESKTOP -#define IF_DESKTOP(x) x +#define IF_DESKTOP(x) x #define IF_NOT_DESKTOP(x) #else #define IF_DESKTOP(x) -#define IF_NOT_DESKTOP(x) x +#define IF_NOT_DESKTOP(x) x +#endif +#define IF_NOT_FEATURE_LZMA_FAST(x) x +#define ENABLE_FEATURE_UNZIP_CDF 1 +#define ENABLE_FEATURE_UNZIP_BZIP2 1 +#define ENABLE_FEATURE_UNZIP_LZMA 1 +#define ENABLE_FEATURE_UNZIP_XZ 1 +#define ENABLE_FEATURE_CLEAN_UP 1 +#define uoff_t unsigned off_t +#define OFF_FMT "ll" + +#define SEAMLESS_COMPRESSION 0 +#if (SEAMLESS_COMPRESSION) +#define ENABLE_FEATURE_SEAMLESS_BZ2 1 +#define ENABLE_FEATURE_SEAMLESS_GZ 1 +#define ENABLE_FEATURE_SEAMLESS_LZMA 1 +#define ENABLE_FEATURE_SEAMLESS_XZ 1 +#define ENABLE_FEATURE_SEAMLESS_Z 1 +#define ENABLE_FEATURE_SEAMLESS_ZSTD 1 +#define IF_FEATURE_SEAMLESS_BZ2(x) x +#define IF_FEATURE_SEAMLESS_XZ(x) x +#define IF_FEATURE_SEAMLESS_ZSTD(x) x #endif -#define IF_NOT_FEATURE_LZMA_FAST(x) x -#define ENABLE_FEATURE_UNZIP_CDF 1 -#define ENABLE_FEATURE_UNZIP_BZIP2 1 -#define ENABLE_FEATURE_UNZIP_LZMA 1 -#define ENABLE_FEATURE_UNZIP_XZ 1 - -#define uoff_t unsigned off_t -#define OFF_FMT "ll" #ifndef _MODE_T_ #define _MODE_T_ @@ -149,8 +162,8 @@ extern unsigned long* bled_cancel_request; #define bb_printf(...) do { if (bled_printf != NULL) bled_printf(__VA_ARGS__); \ else { printf(__VA_ARGS__); putchar('\n'); } } while(0) #define bb_error_msg(...) bb_printf("\nError: " __VA_ARGS__) -#define bb_error_msg_and_die(...) do {bb_error_msg(__VA_ARGS__); xfunc_die();} while(0) -#define bb_error_msg_and_err(...) do {bb_error_msg(__VA_ARGS__); goto err;} while(0) +#define bb_error_msg_and_die(...) do { bb_error_msg(__VA_ARGS__); xfunc_die(); } while(0) +#define bb_error_msg_and_err(...) do { bb_error_msg(__VA_ARGS__); goto err; } while(0) #define bb_perror_msg bb_error_msg #define bb_perror_msg_and_die bb_error_msg_and_die #define bb_simple_error_msg bb_error_msg @@ -167,15 +180,15 @@ static inline void *xrealloc(void *ptr, size_t size) { #define bb_msg_read_error "read error" #define bb_msg_write_error "write error" -#define bb_mode_string(str, mode) "[not implemented]" +#define bb_mode_string(str, mode) str #define bb_make_directory(path, mode, flags) SHCreateDirectoryExU(NULL, path, NULL) -static inline int link(const char *oldpath, const char *newpath) {errno = ENOSYS; return -1;} -static inline int symlink(const char *oldpath, const char *newpath) {errno = ENOSYS; return -1;} -static inline int chown(const char *path, uid_t owner, gid_t group) {errno = ENOSYS; return -1;} -static inline int mknod(const char *pathname, mode_t mode, dev_t dev) {errno = ENOSYS; return -1;} +static inline int link(const char *oldpath, const char *newpath) { errno = ENOSYS; return -1; } +static inline int symlink(const char *oldpath, const char *newpath) { errno = ENOSYS; return -1; } +static inline int chown(const char *path, uid_t owner, gid_t group) { errno = ENOSYS; return -1; } +static inline int mknod(const char *pathname, mode_t mode, dev_t dev) { errno = ENOSYS; return -1; } static inline int utimes64(const char* filename, const struct timeval64 times64[2]) { errno = ENOSYS; return -1; } -static inline int fnmatch(const char *pattern, const char *string, int flags) {return PathMatchSpecA(string, pattern)?0:1;} +static inline int fnmatch(const char *pattern, const char *string, int flags) { return PathMatchSpecA(string, pattern) ? 0 : 1; } static inline pid_t wait(int* status) { *status = 4; return -1; } #define wait_any_nohang wait @@ -283,6 +296,19 @@ static inline struct tm *localtime_r(const time_t *timep, struct tm *result) { #define xzalloc(x) calloc(x, 1) #define malloc_or_warn malloc #define mkdir(x, y) _mkdirU(x) +struct fd_pair { int rd; int wr; }; +void xpipe(int filedes[2]) FAST_FUNC; +#define xpiped_pair(pair) xpipe(&((pair).rd)) +#define xpipe(filedes) _pipe(filedes, 0x1000, _O_BINARY) +#define xlseek lseek +#define xread safe_read +static inline void xmove_fd(int from, int to) +{ + if (from != to) { + _dup2(from, to); + _close(from); + } +} #if defined(_MSC_VER) #define _S_IFBLK 0x3000 diff --git a/src/bled/open_transformer.c b/src/bled/open_transformer.c index f85855eb3f6..2528023f658 100644 --- a/src/bled/open_transformer.c +++ b/src/bled/open_transformer.c @@ -2,7 +2,6 @@ /* * Licensed under GPLv2 or later, see file LICENSE in this source tree. */ - #include "libbb.h" #include "bb_archive.h" @@ -17,12 +16,9 @@ int FAST_FUNC check_signature16(transformer_state_t *xstate, unsigned magic16) uint16_t magic2; if (full_read(xstate->src_fd, &magic2, 2) != 2 || magic2 != magic16) { bb_error_msg("invalid magic"); -#if 0 /* possible future extension */ - if (xstate->check_signature > 1) - xfunc_die(); -#endif return -1; } + xstate->signature_skipped = 2; } return 0; } @@ -101,7 +97,7 @@ void check_errors_in_children(int signo) /* transformer(), more than meets the eye */ #if BB_MMU void FAST_FUNC fork_transformer(int fd, - int check_signature, + int signature_skipped, IF_DESKTOP(long long) int FAST_FUNC (*transformer)(transformer_state_t *xstate) ) #else @@ -115,20 +111,20 @@ void FAST_FUNC fork_transformer(int fd, const char *transform_prog) pid = BB_MMU ? xfork() : xvfork(); if (pid == 0) { /* Child */ - close(fd_pipe.rd); /* we don't want to read from the parent */ + _close(fd_pipe.rd); /* we don't want to read from the parent */ // FIXME: error check? #if BB_MMU { IF_DESKTOP(long long) int r; transformer_state_t xstate; init_transformer_state(&xstate); - xstate.check_signature = check_signature; + xstate.signature_skipped = signature_skipped; xstate.src_fd = fd; xstate.dst_fd = fd_pipe.wr; r = transformer(&xstate); if (ENABLE_FEATURE_CLEAN_UP) { - close(fd_pipe.wr); /* send EOF */ - close(fd); + _close(fd_pipe.wr); /* send EOF */ + _close(fd); } /* must be _exit! bug was actually seen here */ _exit(/*error if:*/ r < 0); @@ -150,7 +146,7 @@ void FAST_FUNC fork_transformer(int fd, const char *transform_prog) } /* parent process */ - close(fd_pipe.wr); /* don't want to write to the child */ + _close(fd_pipe.wr); /* don't want to write to the child */ xmove_fd(fd_pipe.rd, fd); } @@ -159,48 +155,44 @@ void FAST_FUNC fork_transformer(int fd, const char *transform_prog) */ static transformer_state_t *setup_transformer_on_fd(int fd, int fail_if_not_compressed) { - union { - uint8_t b[4]; - uint16_t b16[2]; - uint32_t b32[1]; - } magic; - int offset; transformer_state_t *xstate; - offset = -2; xstate = xzalloc(sizeof(*xstate)); xstate->src_fd = fd; /* .gz and .bz2 both have 2-byte signature, and their * unpack_XXX_stream wants this header skipped. */ - xread(fd, magic.b16, sizeof(magic.b16[0])); + xstate->signature_skipped = 2; + xread(fd, xstate->magic.b16, 2); if (ENABLE_FEATURE_SEAMLESS_GZ - && magic.b16[0] == GZIP_MAGIC + && xstate->magic.b16[0] == GZIP_MAGIC ) { xstate->xformer = unpack_gz_stream; USE_FOR_NOMMU(xstate->xformer_prog = "gunzip";) goto found_magic; } if (ENABLE_FEATURE_SEAMLESS_Z - && magic.b16[0] == COMPRESS_MAGIC + && xstate->magic.b16[0] == COMPRESS_MAGIC ) { xstate->xformer = unpack_Z_stream; USE_FOR_NOMMU(xstate->xformer_prog = "uncompress";) goto found_magic; } if (ENABLE_FEATURE_SEAMLESS_BZ2 - && magic.b16[0] == BZIP2_MAGIC + && xstate->magic.b16[0] == BZIP2_MAGIC ) { xstate->xformer = unpack_bz2_stream; USE_FOR_NOMMU(xstate->xformer_prog = "bunzip2";) goto found_magic; } if (ENABLE_FEATURE_SEAMLESS_XZ - && magic.b16[0] == XZ_MAGIC1 + && xstate->magic.b16[0] == XZ_MAGIC1 ) { - offset = -6; - xread(fd, magic.b32, sizeof(magic.b32[0])); - if (magic.b32[0] == XZ_MAGIC2) { + uint32_t v32; + xstate->signature_skipped = 6; + xread(fd, &xstate->magic.b16[1], 4); + move_from_unaligned32(v32, &xstate->magic.b16[1]); + if (v32 == XZ_MAGIC2) { xstate->xformer = unpack_xz_stream; USE_FOR_NOMMU(xstate->xformer_prog = "unxz";) goto found_magic; @@ -209,9 +201,11 @@ static transformer_state_t *setup_transformer_on_fd(int fd, int fail_if_not_comp if (ENABLE_FEATURE_SEAMLESS_ZSTD && xstate->magic.b16[0] == ZSTD_MAGIC1 ) { - offset = -4; + uint16_t v16; + xstate->signature_skipped = 4; xread(fd, &xstate->magic.b16[1], 2); - if (xstate->magic.b16[1] == ZSTD_MAGIC2) { + move_from_unaligned16(v16, &xstate->magic.b16[1]); + if (v16 == ZSTD_MAGIC2) { xstate->xformer = unpack_zstd_stream; USE_FOR_NOMMU(xstate->xformer_prog = "unzstd";) goto found_magic; @@ -232,19 +226,25 @@ static transformer_state_t *setup_transformer_on_fd(int fd, int fail_if_not_comp */ // USE_FOR_MMU(xstate->xformer = copy_stream;) // USE_FOR_NOMMU(xstate->xformer_prog = "cat";) - /* fall through to seeking bck over bytes we read earlier */ - USE_FOR_NOMMU(found_magic:) + found_magic: + return xstate; +} + +static void fork_transformer_and_free(transformer_state_t *xstate) +{ +# if BB_MMU + fork_transformer_with_no_sig(xstate->src_fd, xstate->xformer); +# else /* NOMMU version of fork_transformer execs * an external unzipper that wants * file position at the start of the file. */ - xlseek(fd, offset, SEEK_CUR); - - USE_FOR_MMU(found_magic:) - /* In MMU case, if magic was found, seeking back is not necessary */ - - return xstate; + xlseek(xstate->src_fd, - xstate->signature_skipped, SEEK_CUR); + xstate->signature_skipped = 0; + fork_transformer_with_sig(xstate->src_fd, xstate->xformer, xstate->xformer_prog); +# endif + free(xstate); } /* Used by e.g. rpm which gives us a fd without filename, @@ -259,21 +259,27 @@ int FAST_FUNC setup_unzip_on_fd(int fd, int fail_if_not_compressed) return 1; } -# if BB_MMU - fork_transformer_with_no_sig(xstate->src_fd, xstate->xformer); -# else - fork_transformer_with_sig(xstate->src_fd, xstate->xformer, xstate->xformer_prog); -# endif - free(xstate); + fork_transformer_and_free(xstate); return 0; } +#if ENABLE_FEATURE_SEAMLESS_LZMA +/* ...and custom version for LZMA */ +void FAST_FUNC setup_lzma_on_fd(int fd) +{ + transformer_state_t *xstate = xzalloc(sizeof(*xstate)); + xstate->src_fd = fd; + xstate->xformer = unpack_lzma_stream; + USE_FOR_NOMMU(xstate->xformer_prog = "unlzma";) + fork_transformer_and_free(xstate); +} +#endif static transformer_state_t *open_transformer(const char *fname, int fail_if_not_compressed) { transformer_state_t *xstate; int fd; - fd = open(fname, O_RDONLY); + fd = _open(fname, O_RDONLY); if (fd < 0) return NULL; @@ -304,14 +310,22 @@ int FAST_FUNC open_zipped(const char *fname, int fail_if_not_compressed) return -1; fd = xstate->src_fd; - if (xstate->xformer) { # if BB_MMU - fork_transformer_with_no_sig(xstate->src_fd, xstate->xformer); + if (xstate->xformer) { + fork_transformer_with_no_sig(fd, xstate->xformer); + } else { + /* the file is not compressed */ + xlseek(fd, - xstate->signature_skipped, SEEK_CUR); + xstate->signature_skipped = 0; + } # else - fork_transformer_with_sig(xstate->src_fd, xstate->xformer, xstate->xformer_prog); + /* NOMMU can't avoid the seek :( */ + xlseek(fd, - xstate->signature_skipped, SEEK_CUR); + xstate->signature_skipped = 0; + if (xstate->xformer) { + fork_transformer_with_sig(fd, xstate->xformer, xstate->xformer_prog); + } /* else: the file is not compressed */ # endif - } - /* else: the file is not compressed */ free(xstate); return fd; @@ -338,13 +352,29 @@ void* FAST_FUNC xmalloc_open_zipped_read_close(const char *fname, size_t *maxsz_ *maxsz_p = xstate->mem_output_size; } } else { - /* File is not compressed */ - image = xmalloc_read(xstate->src_fd, maxsz_p); + /* File is not compressed. + * We already read first few bytes, account for that. + * Example where it happens: + * "modinfo MODULE.ko" (not compressed) + * open("MODULE.ko", O_RDONLY|O_LARGEFILE) = 4 + * read(4, "\177E", 2) = 2 + * fstat64(4, ...) + * mmap(...) + * read(4, "LF\2\1\1\0\0\0\0"... + * ...and we avoided seeking on the fd! :) + */ + image = xmalloc_read_with_initial_buf( + xstate->src_fd, + maxsz_p, + xmemdup(&xstate->magic, xstate->signature_skipped), + xstate->signature_skipped + ); + xstate->signature_skipped = 0; } if (!image) bb_perror_msg("read error from '%s'", fname); - close(xstate->src_fd); + _close(xstate->src_fd); free(xstate); return image; # else diff --git a/src/bled/seek_by_jump.c b/src/bled/seek_by_jump.c index 4fcd99ac80e..232d97e5369 100644 --- a/src/bled/seek_by_jump.c +++ b/src/bled/seek_by_jump.c @@ -2,7 +2,6 @@ /* * Licensed under GPLv2 or later, see file LICENSE in this source tree. */ - #include "libbb.h" #include "bb_archive.h" diff --git a/src/bled/seek_by_read.c b/src/bled/seek_by_read.c index c0fde966085..df234635480 100644 --- a/src/bled/seek_by_read.c +++ b/src/bled/seek_by_read.c @@ -2,7 +2,6 @@ /* * Licensed under GPLv2 or later, see file LICENSE in this source tree. */ - #include "libbb.h" #include "bb_archive.h" diff --git a/src/bled/xxhash.c b/src/bled/xxhash.c index d8e84c68605..052cd522824 100644 --- a/src/bled/xxhash.c +++ b/src/bled/xxhash.c @@ -1,821 +1,18 @@ /* - * xxHash - Fast Hash algorithm - * Copyright (c) Yann Collet, Facebook, Inc. + * xxHash - Extremely Fast Hash algorithm + * Copyright (c) Yann Collet - Meta Platforms, Inc * - * You can contact the author at : - * - xxHash homepage: http://www.xxhash.com - * - xxHash source repository : https://github.com/Cyan4973/xxHash - * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. -*/ - - -/* ************************************* -* Tuning parameters -***************************************/ -/*!XXH_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. - * It can generate buggy code on targets which do not support unaligned memory accesses. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://stackoverflow.com/a/32095106/646947 for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) || \ - defined(__ICCARM__) -# define XXH_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -/*!XXH_ACCEPT_NULL_INPUT_POINTER : - * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. - * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. - * By default, this option is disabled. To enable it, uncomment below define : - */ -/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ - -/*!XXH_FORCE_NATIVE_FORMAT : - * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. - * Results are therefore identical for little-endian and big-endian CPU. - * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. - * Should endian-independence be of no importance for your application, you may set the #define below to 1, - * to improve speed for Big-endian CPU. - * This option has no impact on Little_Endian CPU. */ -#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ -# define XXH_FORCE_NATIVE_FORMAT 0 -#endif -/*!XXH_FORCE_ALIGN_CHECK : - * This is a minor performance trick, only useful with lots of very small keys. - * It means : check for aligned/unaligned input. - * The check costs one initial branch per hash; set to 0 when the input data - * is guaranteed to be aligned. +/* + * xxhash.c instantiates functions defined in xxhash.h */ -#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ -# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_FORCE_ALIGN_CHECK 0 -# else -# define XXH_FORCE_ALIGN_CHECK 1 -# endif -#endif - -/* ************************************* -* Includes & Memory related functions -***************************************/ -/* Modify the local functions below should you wish to use some other memory routines */ -/* for ZSTD_malloc(), ZSTD_free() */ -#define ZSTD_DEPS_NEED_MALLOC -#include "zstd_deps.h" /* size_t, ZSTD_malloc, ZSTD_free, ZSTD_memcpy */ -static void* XXH_malloc(size_t s) { return ZSTD_malloc(s); } -static void XXH_free (void* p) { ZSTD_free(p); } -static void* XXH_memcpy(void* dest, const void* src, size_t size) { return ZSTD_memcpy(dest,src,size); } +#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ +#define XXH_IMPLEMENTATION /* access definitions */ #include "xxhash.h" - - -/* ************************************* -* Compiler Specific Options -***************************************/ -#include "zstd_compiler.h" - - -/* ************************************* -* Basic Types -***************************************/ -#include "zstd_mem.h" /* BYTE, U32, U64, size_t */ - -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) - -/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ -static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } -static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } - -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign; - -static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } - -#else - -/* portable and safe solution. Generally efficient. - * see : http://stackoverflow.com/a/32095106/646947 - */ - -static U32 XXH_read32(const void* memPtr) -{ - U32 val; - ZSTD_memcpy(&val, memPtr, sizeof(val)); - return val; -} - -static U64 XXH_read64(const void* memPtr) -{ - U64 val; - ZSTD_memcpy(&val, memPtr, sizeof(val)); - return val; -} - -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ - - -/* **************************************** -* Compiler-specific Functions and Macros -******************************************/ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -# define XXH_rotl64(x,r) _rotl64(x,r) -#else -#if defined(__ICCARM__) -# include <intrinsics.h> -# define XXH_rotl32(x,r) __ROR(x,(32 - r)) -#else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) -#endif -# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) -#endif - -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap32 _byteswap_ulong -# define XXH_swap64 _byteswap_uint64 -#elif GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -# define XXH_swap64 __builtin_bswap64 -#else -static U32 XXH_swap32 (U32 x) -{ - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); -} -static U64 XXH_swap64 (U64 x) -{ - return ((x << 56) & 0xff00000000000000ULL) | - ((x << 40) & 0x00ff000000000000ULL) | - ((x << 24) & 0x0000ff0000000000ULL) | - ((x << 8) & 0x000000ff00000000ULL) | - ((x >> 8) & 0x00000000ff000000ULL) | - ((x >> 24) & 0x0000000000ff0000ULL) | - ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL); -} -#endif - - -/* ************************************* -* Architecture Macros -***************************************/ -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; - -/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ -#ifndef XXH_CPU_LITTLE_ENDIAN - static const int g_one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) -#endif - - -/* *************************** -* Memory reads -*****************************/ -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE_TEMPLATE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); - else - return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); -} - -FORCE_INLINE_TEMPLATE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE32_align(ptr, endian, XXH_unaligned); -} - -static U32 XXH_readBE32(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); -} - -FORCE_INLINE_TEMPLATE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); - else - return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); -} - -FORCE_INLINE_TEMPLATE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE64_align(ptr, endian, XXH_unaligned); -} - -static U64 XXH_readBE64(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); -} - - -/* ************************************* -* Macros -***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/* ************************************* -* Constants -***************************************/ -static const U32 PRIME32_1 = 2654435761U; -static const U32 PRIME32_2 = 2246822519U; -static const U32 PRIME32_3 = 3266489917U; -static const U32 PRIME32_4 = 668265263U; -static const U32 PRIME32_5 = 374761393U; - -static const U64 PRIME64_1 = 11400714785074694791ULL; -static const U64 PRIME64_2 = 14029467366897019727ULL; -static const U64 PRIME64_3 = 1609587929392839161ULL; -static const U64 PRIME64_4 = 9650029242287828579ULL; -static const U64 PRIME64_5 = 2870177450012600261ULL; - -XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } - - -/* ************************** -* Utils -****************************/ -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState) -{ - ZSTD_memcpy(dstState, srcState, sizeof(*dstState)); -} - -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState) -{ - ZSTD_memcpy(dstState, srcState, sizeof(*dstState)); -} - - -/* *************************** -* Simple Hash Functions -*****************************/ - -static U32 XXH32_round(U32 seed, U32 input) -{ - seed += input * PRIME32_2; - seed = XXH_rotl32(seed, 13); - seed *= PRIME32_1; - return seed; -} - -FORCE_INLINE_TEMPLATE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)16; - } -#endif - - if (len>=16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - - do { - v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; - v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; - v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; - v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; - } while (p<=limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } else { - h32 = seed + PRIME32_5; - } - - h32 += (U32) len; - - while (p+4<=bEnd) { - h32 += XXH_get32bits(p) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } - - while (p<bEnd) { - h32 += (*p) * PRIME32_5; - h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; - p++; - } - - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH32_CREATESTATE_STATIC(state); - XXH32_reset(state, seed); - XXH32_update(state, input, len); - return XXH32_digest(state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - -static U64 XXH64_round(U64 acc, U64 input) -{ - acc += input * PRIME64_2; - acc = XXH_rotl64(acc, 31); - acc *= PRIME64_1; - return acc; -} - -static U64 XXH64_mergeRound(U64 acc, U64 val) -{ - val = XXH64_round(0, val); - acc ^= val; - acc = acc * PRIME64_1 + PRIME64_4; - return acc; -} - -FORCE_INLINE_TEMPLATE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)32; - } -#endif - - if (len>=32) { - const BYTE* const limit = bEnd - 32; - U64 v1 = seed + PRIME64_1 + PRIME64_2; - U64 v2 = seed + PRIME64_2; - U64 v3 = seed + 0; - U64 v4 = seed - PRIME64_1; - - do { - v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; - v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; - v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; - v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; - } while (p<=limit); - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - - } else { - h64 = seed + PRIME64_5; - } - - h64 += (U64) len; - - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_get64bits(p)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p<bEnd) { - h64 ^= (*p) * PRIME64_5; - h64 = XXH_rotl64(h64, 11) * PRIME64_1; - p++; - } - - h64 ^= h64 >> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - - -XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH64_CREATESTATE_STATIC(state); - XXH64_reset(state, seed); - XXH64_update(state, input, len); - return XXH64_digest(state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - -/* ************************************************** -* Advanced Hash Functions -****************************************************/ - -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) -{ - return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} - -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) -{ - return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} - - -/*** Hash feed ***/ - -XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) -{ - XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - ZSTD_memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ - state.v1 = seed + PRIME32_1 + PRIME32_2; - state.v2 = seed + PRIME32_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME32_1; - ZSTD_memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; -} - - -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) -{ - XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - ZSTD_memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ - state.v1 = seed + PRIME64_1 + PRIME64_2; - state.v2 = seed + PRIME64_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME64_1; - ZSTD_memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; -} - - -FORCE_INLINE_TEMPLATE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len_32 += (unsigned)len; - state->large_len |= (len>=16) | (state->total_len_32>=16); - - if (state->memsize + len < 16) { /* fill in tmp buffer */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); - state->memsize += (unsigned)len; - return XXH_OK; - } - - if (state->memsize) { /* some data left from previous update */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); - { const U32* p32 = state->mem32; - state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; - state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; - state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; - } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - - do { - v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; - v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; - v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; - v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } - - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE_TEMPLATE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem32; - const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; - U32 h32; - - if (state->large_len) { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } else { - h32 = state->v3 /* == seed */ + PRIME32_5; - } - - h32 += state->total_len_32; - - while (p+4<=bEnd) { - h32 += XXH_readLE32(p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p<bEnd) { - h32 += (*p) * PRIME32_5; - h32 = XXH_rotl32(h32, 11) * PRIME32_1; - p++; - } - - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_digest_endian(state_in, XXH_littleEndian); - else - return XXH32_digest_endian(state_in, XXH_bigEndian); -} - - - -/* **** XXH64 **** */ - -FORCE_INLINE_TEMPLATE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 32) { /* fill in tmp buffer */ - if (input != NULL) { - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); - } - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) { /* tmp buffer is full */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); - state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); - state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); - state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); - state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); - p += 32-state->memsize; - state->memsize = 0; - } - - if (p+32 <= bEnd) { - const BYTE* const limit = bEnd - 32; - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; - - do { - v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; - v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; - v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; - v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } - - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH64_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE_TEMPLATE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem64; - const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; - U64 h64; - - if (state->total_len >= 32) { - U64 const v1 = state->v1; - U64 const v2 = state->v2; - U64 const v3 = state->v3; - U64 const v4 = state->v4; - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - } else { - h64 = state->v3 + PRIME64_5; - } - - h64 += (U64) state->total_len; - - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p<bEnd) { - h64 ^= (*p) * PRIME64_5; - h64 = XXH_rotl64(h64, 11) * PRIME64_1; - p++; - } - - h64 ^= h64 >> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - - -XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_digest_endian(state_in, XXH_littleEndian); - else - return XXH64_digest_endian(state_in, XXH_bigEndian); -} - - -/* ************************** -* Canonical representation -****************************/ - -/*! Default XXH result types are basic unsigned 32 and 64 bits. -* The canonical representation follows human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. -*/ - -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); - ZSTD_memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); - ZSTD_memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) -{ - return XXH_readBE32(src); -} - -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) -{ - return XXH_readBE64(src); -} diff --git a/src/bled/xxhash.h b/src/bled/xxhash.h index 8865cfd819e..e59e44267c1 100644 --- a/src/bled/xxhash.h +++ b/src/bled/xxhash.h @@ -1,276 +1,7020 @@ /* * xxHash - Extremely Fast Hash algorithm * Header File - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet - Meta Platforms, Inc * - * You can contact the author at : - * - xxHash source repository : https://github.com/Cyan4973/xxHash - * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. -*/ + */ -/* Notice extracted from xxHash homepage : +/* Local adaptations for Zstandard */ -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. - -A 64-bits version, named XXH64, is available since r35. -It offers much better speed, but for 64-bits applications only. -Name Speed on 64 bits Speed on 32 bits -XXH64 13.8 GB/s 1.9 GB/s -XXH32 6.8 GB/s 6.0 GB/s -*/ +#ifndef XXH_NO_XXH3 +# define XXH_NO_XXH3 +#endif +#ifndef XXH_NAMESPACE +# define XXH_NAMESPACE ZSTD_ +#endif -#ifndef XXHASH_H_5627135585666179 -#define XXHASH_H_5627135585666179 1 +/*! + * @mainpage xxHash + * + * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed + * limits. + * + * It is proposed in four flavors, in three families: + * 1. @ref XXH32_family + * - Classic 32-bit hash function. Simple, compact, and runs on almost all + * 32-bit and 64-bit systems. + * 2. @ref XXH64_family + * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most + * 64-bit systems (but _not_ 32-bit systems). + * 3. @ref XXH3_family + * - Modern 64-bit and 128-bit hash function family which features improved + * strength and performance across the board, especially on smaller data. + * It benefits greatly from SIMD and 64-bit without requiring it. + * + * Benchmarks + * --- + * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. + * The open source benchmark program is compiled with clang v10.0 using -O3 flag. + * + * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | + * | -------------------- | ------- | ----: | ---------------: | ------------------: | + * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | + * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | + * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | + * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | + * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | + * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | + * | RAM sequential read | | N/A | 28.0 GB/s | N/A | + * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | + * | City64 | | 64 | 22.0 GB/s | 76.6 | + * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | + * | City128 | | 128 | 21.7 GB/s | 57.7 | + * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | + * | XXH64() | | 64 | 19.4 GB/s | 71.0 | + * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | + * | Mum | | 64 | 18.0 GB/s | 67.0 | + * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | + * | XXH32() | | 32 | 9.7 GB/s | 71.9 | + * | City32 | | 32 | 9.1 GB/s | 66.0 | + * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | + * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | + * | SipHash* | | 64 | 3.0 GB/s | 43.2 | + * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | + * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | + * | FNV64 | | 64 | 1.2 GB/s | 62.7 | + * | Blake2* | | 256 | 1.1 GB/s | 5.1 | + * | SHA1* | | 160 | 0.8 GB/s | 5.6 | + * | MD5* | | 128 | 0.6 GB/s | 7.8 | + * @note + * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, + * even though it is mandatory on x64. + * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic + * by modern standards. + * - Small data velocity is a rough average of algorithm's efficiency for small + * data. For more accurate information, see the wiki. + * - More benchmarks and strength tests are found on the wiki: + * https://github.com/Cyan4973/xxHash/wiki + * + * Usage + * ------ + * All xxHash variants use a similar API. Changing the algorithm is a trivial + * substitution. + * + * @pre + * For functions which take an input and length parameter, the following + * requirements are assumed: + * - The range from [`input`, `input + length`) is valid, readable memory. + * - The only exception is if the `length` is `0`, `input` may be `NULL`. + * - For C++, the objects must have the *TriviallyCopyable* property, as the + * functions access bytes directly as if it was an array of `unsigned char`. + * + * @anchor single_shot_example + * **Single Shot** + * + * These functions are stateless functions which hash a contiguous block of memory, + * immediately returning the result. They are the easiest and usually the fastest + * option. + * + * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() + * + * @code{.c} + * #include <string.h> + * #include "xxhash.h" + * + * // Example for a function which hashes a null terminated string with XXH32(). + * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) + * { + * // NULL pointers are only valid if the length is zero + * size_t length = (string == NULL) ? 0 : strlen(string); + * return XXH32(string, length, seed); + * } + * @endcode + * + * + * @anchor streaming_example + * **Streaming** + * + * These groups of functions allow incremental hashing of unknown size, even + * more than what would fit in a size_t. + * + * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() + * + * @code{.c} + * #include <stdio.h> + * #include <assert.h> + * #include "xxhash.h" + * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). + * XXH64_hash_t hashFile(FILE* f) + * { + * // Allocate a state struct. Do not just use malloc() or new. + * XXH3_state_t* state = XXH3_createState(); + * assert(state != NULL && "Out of memory!"); + * // Reset the state to start a new hashing session. + * XXH3_64bits_reset(state); + * char buffer[4096]; + * size_t count; + * // Read the file in chunks + * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { + * // Run update() as many times as necessary to process the data + * XXH3_64bits_update(state, buffer, count); + * } + * // Retrieve the finalized hash. This will not change the state. + * XXH64_hash_t result = XXH3_64bits_digest(state); + * // Free the state. Do not use free(). + * XXH3_freeState(state); + * return result; + * } + * @endcode + * + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * + * @anchor canonical_representation_example + * **Canonical Representation** + * + * The default return values from XXH functions are unsigned 32, 64 and 128 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + * + * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(), + * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(), + * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(), + * + * @code{.c} + * #include <stdio.h> + * #include "xxhash.h" + * + * // Example for a function which prints XXH32_hash_t in human readable format + * void printXxh32(XXH32_hash_t hash) + * { + * XXH32_canonical_t cano; + * XXH32_canonicalFromHash(&cano, hash); + * size_t i; + * for(i = 0; i < sizeof(cano.digest); ++i) { + * printf("%02x", cano.digest[i]); + * } + * printf("\n"); + * } + * + * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t + * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano) + * { + * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano); + * return hash; + * } + * @endcode + * + * + * @file xxhash.h + * xxHash prototypes and implementation + */ +#if defined (__cplusplus) +extern "C" { +#endif /* **************************** -* Definitions -******************************/ -#include "zstd_deps.h" -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + * INLINE mode + ******************************/ +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Gives access to internal state declaration, required for static allocation. + * + * Incompatible with dynamic linking, due to risks of ABI changes. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #include "xxhash.h" + * @endcode + */ +# define XXH_STATIC_LINKING_ONLY +/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ +/*! + * @brief Gives access to internal definitions. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #define XXH_IMPLEMENTATION + * #include "xxhash.h" + * @endcode + */ +# define XXH_IMPLEMENTATION +/* Do not undef XXH_IMPLEMENTATION for Doxygen */ -/* **************************** -* API modifier -******************************/ -/** XXH_PRIVATE_API -* This is useful if you want to include xxhash functions in `static` mode -* in order to inline them, and remove their symbol from the public list. -* Methodology : -* #define XXH_PRIVATE_API -* #include "xxhash.h" -* `xxhash.c` is automatically included. -* It's not useful to compile and link it as a separate module anymore. -*/ -#ifdef XXH_PRIVATE_API +/*! + * @brief Exposes the implementation and marks all functions as `inline`. + * + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * @code{.c} + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * @endcode + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +# define XXH_INLINE_ALL +# undef XXH_INLINE_ALL +/*! + * @brief Exposes the implementation without marking functions as inline. + */ +# define XXH_PRIVATE_API +# undef XXH_PRIVATE_API +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API # if defined(__GNUC__) # define XXH_PUBLIC_API static __inline __attribute__((unused)) -# elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # define XXH_PUBLIC_API static inline # elif defined(_MSC_VER) # define XXH_PUBLIC_API static __inline # else -# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static # endif -#else -# define XXH_PUBLIC_API /* do nothing */ -#endif /* XXH_PRIVATE_API */ -/*!XXH_NAMESPACE, aka Namespace Emulation : + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE -If you want to include _and expose_ xxHash functions from within your own library, -but also want to avoid symbol collisions with another library which also includes xxHash, + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ -you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library -with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +/*! @brief Marks a global symbol. */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif -Note that no change is required within the calling program as long as it includes `xxhash.h` : -regular symbol name will be automatically translated by this header. -*/ #ifdef XXH_NAMESPACE # define XXH_CAT(A,B) A##B # define XXH_NAME2(A,B) XXH_CAT(A,B) -# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) -# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) # define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) # define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) -# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) # define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) -# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) # define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) -# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) # define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) -# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) # define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) -# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) # define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) -# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) # define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) -# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) # define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) # define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Compiler specifics +***************************************/ + +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif #endif +#if defined (__GNUC__) +# define XXH_CONSTF __attribute__((const)) +# define XXH_PUREF __attribute__((pure)) +# define XXH_MALLOCF __attribute__((malloc)) +#else +# define XXH_CONSTF /* disable */ +# define XXH_PUREF +# define XXH_MALLOCF +#endif /* ************************************* * Version ***************************************/ #define XXH_VERSION_MAJOR 0 -#define XXH_VERSION_MINOR 6 +#define XXH_VERSION_MINOR 8 #define XXH_VERSION_RELEASE 2 +/*! @brief Version number, encoded as two digits each */ #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) -XXH_PUBLIC_API unsigned XXH_versionNumber (void); + +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return @ref XXH_VERSION_NUMBER of the invoked library. + */ +XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); /* **************************** -* Simple Hash Functions +* Common basic types ******************************/ -typedef unsigned int XXH32_hash_t; -typedef unsigned long long XXH64_hash_t; +#include <stddef.h> /* size_t */ +/*! + * @brief Exit code for the streaming API. + */ +typedef enum { + XXH_OK = 0, /*!< OK */ + XXH_ERROR /*!< Error */ +} XXH_errorcode; -XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); -XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show <stdint.h> include */ /*! -XXH32() : - Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". - The memory between input & input+length must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s -XXH64() : - Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". - "seed" can be used to alter the result predictably. - This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark). -*/ + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include <inttypes.h> +# else +# include <stdint.h> +# endif + typedef uint32_t XXH32_hash_t; -/* **************************** -* Streaming Hash Functions -******************************/ -typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ -typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +#else +# include <limits.h> +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# elif ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +#endif -/*! State allocation, compatible with dynamic libraries */ +/*! + * @} + * + * @defgroup XXH32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that the @ref XXH3_family provides competitive speed for both 32-bit + * and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families + * @see @ref XXH32_impl for implementation details + * @{ + */ -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit xxHash32 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +#ifndef XXH_NO_STREAM +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * @return An allocated pointer of @ref XXH32_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH32_freeState(). + */ +XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH32_createState(). + * + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); -/* hash streaming */ +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH32_update(). + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); -XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); -XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 32-bit xxHash32 value from that state. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ -/* -These functions generate the xxHash of an input provided in multiple segments. -Note that, for small input, they are slower than single-call functions, due to state management. -For small input, prefer `XXH32()` and `XXH64()` . +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; -XXH state must first be allocated, using XXH*_createState() . +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); -Start a new hash by initializing state with a seed, using XXH*_reset(). +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); -Then, feed the hash state by calling XXH*_update() as many times as necessary. -Obviously, input must be allocated and read accessible. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. -Finally, a hash value can be produced anytime, by using XXH*_digest(). -This function returns the nn-bits hash as an int or long long. +/*! @cond Doxygen ignores this part */ +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ -It's still possible to continue inserting input into the hash state after a digest, -and generate some new hashes later on, by calling again XXH*_digest(). +/*! @cond Doxygen ignores this part */ +/* + * C23 __STDC_VERSION__ number hasn't been specified yet. For now + * leave as `201711L` (C17 + 1). + * TODO: Update to correct value when its been specified. + */ +#define XXH_C23_VN 201711L +/*! @endcond */ -When done, free XXH state space if it was allocated dynamically. -*/ +/*! @cond Doxygen ignores this part */ +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ +/*! @cond Doxygen ignores this part */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ -/* ************************** -* Utils -****************************/ -#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */ -# define restrict /* disable restrict */ +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute + * introduced in CPP17 and C23. + * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough + */ +#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) +#else +# define XXH_FALLTHROUGH /* fallthrough */ #endif +/*! @endcond */ -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state); -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state); +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_NOESCAPE for annotated pointers in public API. + * https://clang.llvm.org/docs/AttributeReference.html#noescape + * As of writing this, only supported by clang. + */ +#if XXH_HAS_ATTRIBUTE(noescape) +# define XXH_NOESCAPE __attribute__((noescape)) +#else +# define XXH_NOESCAPE +#endif +/*! @endcond */ -/* ************************** -* Canonical representation -****************************/ -/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. -* The canonical representation uses human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. -*/ -typedef struct { unsigned char digest[4]; } XXH32_canonical_t; -typedef struct { unsigned char digest[8]; } XXH64_canonical_t; +/*! + * @} + * @ingroup public + * @{ + */ -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include <stdint.h> */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include <inttypes.h> +# else +# include <stdint.h> +# endif + typedef uint64_t XXH64_hash_t; +#else +# include <limits.h> +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); +/*! + * @} + * + * @defgroup XXH64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ -#endif /* XXHASH_H_5627135585666179 */ +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit xxHash64 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +/*! + * @brief Allocates an @ref XXH64_state_t. + * + * @return An allocated pointer of @ref XXH64_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH64_freeState(). + */ +XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); +/*! + * @brief Frees an @ref XXH64_state_t. + * + * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH64_createState(). + */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); -/* ================================================================================================ - This section contains definitions which are not guaranteed to remain stable. - They may change in future versions, becoming incompatible with a different version of the library. - They shall only be used with static linking. - Never use these definitions in association with dynamic linking ! -=================================================================================================== */ -#if !defined(XXH_STATIC_H_3543687687345) -#define XXH_STATIC_H_3543687687345 - -/* These definitions are only meant to allow allocation of XXH state - statically, on stack, or in a struct for example. - Do not use members directly. */ - - struct XXH32_state_s { - unsigned total_len_32; - unsigned large_len; - unsigned v1; - unsigned v2; - unsigned v3; - unsigned v4; - unsigned mem32[4]; /* buffer defined as U32 for alignment */ - unsigned memsize; - unsigned reserved; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH32_state_t */ - - struct XXH64_state_s { - unsigned long long total_len; - unsigned long long v1; - unsigned long long v2; - unsigned long long v3; - unsigned long long v4; - unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ - unsigned memsize; - unsigned reserved[2]; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH64_state_t */ - - -# ifdef XXH_PRIVATE_API -# include "xxhash.c" /* include xxhash functions as `static`, for inlining */ -# endif +/*! + * @brief Copies one @ref XXH64_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +/*! + * @brief Resets an @ref XXH64_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH64_update(). + */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH64_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH64_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 64-bit xxHash64 value from that state. + * + * @note + * Calling XXH64_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH64_hash_t. + */ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; + +/*! + * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. + * + * @param dst The @ref XXH64_canonical_t pointer to be stored to. + * @param hash The @ref XXH64_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); + +/*! + * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. + * + * @param src The @ref XXH64_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); -#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */ +#ifndef XXH_NO_XXH3 +/*! + * @} + * ************************************************************************ + * @defgroup XXH3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 + * at competitive speeds, even without vector support. Further details are + * explained in the implementation. + * + * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD + * implementations for many common platforms: + * - AVX512 + * - AVX2 + * - SSE2 + * - ARM NEON + * - WebAssembly SIMD128 + * - POWER8 VSX + * - s390x ZVector + * This can be controlled via the @ref XXH_VECTOR macro, but it automatically + * selects the best version according to predefined macros. For the x86 family, an + * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generate exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ +/*! + * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see + * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/*! + * @brief Calculates 64-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The opaque state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH3_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits()`. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits_withSeed()`. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * `secret` is referenced, it _must outlive_ the hash streaming session. + * + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 64-bit hash value from that state. + * + * @note + * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +/*! + * @brief Calculates 128-bit unseeded variant of XXH3 of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead + * for shorter inputs. + * + * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); +/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 128-bit hash value from that state. + * + * @note + * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * @brief Check equality of two XXH128_hash_t values + * + * @param h1 The 128-bit hash value. + * @param h2 Another 128-bit hash value. + * + * @return `1` if `h1` and `h2` are equal. + * @return `0` if they are not. + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * @brief Compares two @ref XXH128_hash_t + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * @param h128_1 Left-hand side value + * @param h128_2 Right-hand side value + * + * @return >0 if @p h128_1 > @p h128_2 + * @return =0 if @p h128_1 == @p h128_2 + * @return <0 if @p h128_1 < @p h128_2 + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; + + +/*! + * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. + * + * @param dst The @ref XXH128_canonical_t pointer to be stored to. + * @param hash The @ref XXH128_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); + +/*! + * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. + * + * @param src The @ref XXH128_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); + + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include <stdalign.h> +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @internal + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) \ + do { \ + XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ + tmp_xxh3_state_ptr->seed = 0; \ + tmp_xxh3_state_ptr->extSecret = NULL; \ + } while(0) + + +/*! + * @brief Calculates the 128-bit hash of @p data using XXH3. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p len is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 128-bit XXH3 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/*! + * @brief Derive a high-entropy secret from any user-defined content, named customSeed. + * + * @param secretBuffer A writable buffer for derived high-entropy secret data. + * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_DEFAULT_SIZE. + * @param customSeed A user-defined content. + * @param customSeedSize Size of customSeed, in bytes. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection + * than 64-bit seed, as it becomes much more difficult for an external actor to + * guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @p secretSize into an + * already allocated buffer @p secretBuffer. + * + * The generated secret can then be used with any `*_withSecret()` variant. + * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), + * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can + * be employed to ensure proper quality. + * + * @p customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch + * of zeroes. The resulting `secret` will nonetheless provide all required qualities. + * + * @pre + * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN + * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + * + * Example code: + * @code{.c} + * #include <stdio.h> + * #include <stdlib.h> + * #include <string.h> + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Hashes argv[2] using the entropy from argv[1]. + * int main(int argc, char* argv[]) + * { + * char secret[XXH3_SECRET_SIZE_MIN]; + * if (argv != 3) { return 1; } + * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); + * XXH64_hash_t h = XXH3_64bits_withSecret( + * argv[2], strlen(argv[2]), + * secret, sizeof(secret) + * ); + * printf("%016llx\n", (unsigned long long) h); + * } + * @endcode + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); + +/*! + * @brief Generate the same secret as the _withSeed() variants. + * + * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes + * @param seed The 64-bit seed to alter the hash result predictably. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * + * Example C++ `std::string` hash class: + * @code{.cpp} + * #include <string> + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Slow, seeds each time + * class HashSlow { + * XXH64_hash_t seed; + * public: + * HashSlow(XXH64_hash_t s) : seed{s} {} + * size_t operator()(const std::string& x) const { + * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; + * } + * }; + * // Fast, caches the seeded secret for future uses. + * class HashFast { + * unsigned char secret[XXH3_SECRET_SIZE_MIN]; + * public: + * HashFast(XXH64_hash_t s) { + * XXH3_generateSecret_fromSeed(secret, seed); + * } + * size_t operator()(const std::string& x) const { + * return size_t{ + * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) + * }; + * } + * }; + * @endcode + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); + +/*! + * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * These variants generate hash values using either + * @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes) + * or @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @p secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, via its impact to the seed. + * + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param input The block of data to be hashed, at least @p len bytes in size. + * @param length The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#ifndef XXH_NO_STREAM +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#endif /* !XXH_NO_STREAM */ + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_SIZE_OPT + * @brief Controls how much xxHash optimizes for size. + * + * xxHash, when compiled, tends to result in a rather large binary size. This + * is mostly due to heavy usage to forced inlining and constant folding of the + * @ref XXH3_family to increase performance. + * + * However, some developers prefer size over speed. This option can + * significantly reduce the size of the generated code. When using the `-Os` + * or `-Oz` options on GCC or Clang, this is defined to 1 by default, + * otherwise it is defined to 0. + * + * Most of these size optimizations can be controlled manually. + * + * This is a number from 0-2. + * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed + * comes first. + * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more + * conservative and disables hacks that increase code size. It implies the + * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, + * and @ref XXH3_NEON_LANES == 8 if they are not already defined. + * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. + * Performance may cry. For example, the single shot functions just use the + * streaming API. + */ +# define XXH_SIZE_OPT 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips + * which are platforms known to offer good unaligned memory accesses performance. + * + * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if + * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH3_INLINE_SECRET + * @brief Determines whether to inline the XXH3 withSecret code. + * + * When the secret size is known, the compiler can improve the performance + * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). + * + * However, if the secret size is not known, it doesn't have any benefit. This + * happens when xxHash is compiled into a global symbol. Therefore, if + * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. + * + * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers + * that are *sometimes* force inline on -Og, and it is impossible to automatically + * detect this optimization level. + */ +# define XXH3_INLINE_SECRET 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ + +/*! + * @def XXH_NO_STREAM + * @brief Disables the streaming API. + * + * When xxHash is not inlined and the streaming functions are not used, disabling + * the streaming functions can improve code size significantly, especially with + * the @ref XXH3_family which tends to make constant folded copies of itself. + */ +# define XXH_NO_STREAM +# undef XXH_NO_STREAM /* don't actually */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for GCC + * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy + * which for some reason does unaligned loads. */ +# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_SIZE_OPT + /* default to 1 for -Os or -Oz */ +# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) +# define XXH_SIZE_OPT 1 +# else +# define XXH_SIZE_OPT 0 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ + /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ +# if XXH_SIZE_OPT >= 1 || \ + defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH3_INLINE_SECRET +# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ + || !defined(XXH_INLINE_ALL) +# define XXH3_INLINE_SECRET 0 +# else +# define XXH3_INLINE_SECRET 1 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +#if defined(XXH_NO_STREAM) +/* nothing */ +#elif defined(XXH_NO_STDLIB) + +/* When requesting to disable any mention of stdlib, + * the library loses the ability to invoked malloc / free. + * In practice, it means that functions like `XXH*_createState()` + * will always fail, and return NULL. + * This flag is useful in situations where + * xxhash.h is integrated into some kernel, embedded or limited environment + * without access to dynamic allocation. + */ + +static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } +static void XXH_free(void* p) { (void)p; } + +#else + +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include <stdlib.h> + +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ +static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); } + +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } + +#endif /* XXH_NO_STDLIB */ + +#include <string.h> + +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +#include <limits.h> /* ULLONG_MAX */ + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + +#if XXH3_INLINE_SECRET +# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE +#else +# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE +#endif + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include <assert.h> /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# if defined(__INTEL_COMPILER) +# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) +# else +# define XXH_ASSERT(c) XXH_ASSUME(c) +# endif +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* Specifically for NEON vectors which use the "w" constraint, on + * Clang. */ +#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) +# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) +#else +# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include <inttypes.h> +# else +# include <stdint.h> +# endif + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32; + return *((const xxh_unalign32*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + + + +/* + * C23 and future versions have standard "unreachable()". + * Once it has been implemented reliably we can add it as an + * additional case: + * + * ``` + * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) + * # include <stddef.h> + * # ifdef unreachable + * # define XXH_UNREACHABLE() unreachable() + * # endif + * #endif + * ``` + * + * Note C++23 also has std::unreachable() which can be detected + * as follows: + * ``` + * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) + * # include <utility> + * # define XXH_UNREACHABLE() std::unreachable() + * #endif + * ``` + * NB: `__cpp_lib_unreachable` is defined in the `<version>` header. + * We don't use that as including `<utility>` in `extern "C"` blocks + * doesn't work on GCC12 + */ + +#if XXH_HAS_BUILTIN(__builtin_unreachable) +# define XXH_UNREACHABLE() __builtin_unreachable() + +#elif defined(_MSC_VER) +# define XXH_UNREACHABLE() __assume(0) + +#else +# define XXH_UNREACHABLE() +#endif + +#if XXH_HAS_BUILTIN(__builtin_assume) +# define XXH_ASSUME(c) __builtin_assume(c) +#else +# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup XXH32_impl XXH32 implementation + * @ingroup impl + * + * Details on the XXH32 implementation. + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing + * the loop. NEON is only faster on the A53, and with the newer cores, it is less + * than half the speed. + * + * Additionally, this is used on WASM SIMD128 because it JITs to the same + * SIMD instructions and has the same issue. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param hash The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 hash) +{ + hash ^= hash >> 15; + hash *= XXH_PRIME32_2; + hash ^= hash >> 13; + hash *= XXH_PRIME32_3; + hash ^= hash >> 16; + return hash; +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + * @see XXH64_finalize(). + */ +static XXH_PUREF xxh_u32 +XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + hash += (*ptr++) * XXH_PRIME32_5; \ + hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(hash); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 4: XXH_PROCESS4; + return XXH32_avalanche(hash); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 0: return XXH32_avalanche(hash); + } + XXH_ASSERT(0); + return hash; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64; + return *((const xxh_unalign64*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup XXH64_impl XXH64 implementation + * @ingroup impl + * + * Details on the XXH64 implementation. + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +/*! @copydoc XXH32_round */ +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; +#if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * DISABLE AUTOVECTORIZATION: + * A compiler fence is used to prevent GCC and Clang from + * autovectorizing the XXH64 loop (pragmas and attributes don't work for some + * reason) without globally disabling AVX512. + * + * Autovectorization of XXH64 tends to be detrimental, + * though the exact outcome may change depending on exact cpu and compiler version. + * For information, it has been reported as detrimental for Skylake-X, + * but possibly beneficial for Zen4. + * + * The default is to disable auto-vectorization, + * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +/*! @copydoc XXH32_avalanche */ +static xxh_u64 XXH64_avalanche(xxh_u64 hash) +{ + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-31 bytes of @p ptr. + * + * There may be up to 31 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 32. + * @param align Whether @p ptr is aligned. + * @return The finalized hash + * @see XXH32_finalize(). + */ +static XXH_PUREF xxh_u64 +XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + hash ^= k1; + hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + hash ^= (*ptr++) * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(hash); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH64(). + * + * @param input , len , seed Directly passed from @ref XXH64(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (input<limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + + } else { + h64 = seed + XXH_PRIME64_5; + } + + h64 += (xxh_u64) len; + + return XXH64_finalize(h64, input, len, align); +} + + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64 (XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8*)input, len); + return XXH64_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); + +#endif +} + +/******* Hash Streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH64_family*/ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup XXH3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ + || (defined (__clang__)) \ + || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ + || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) +/* + * There are a LOT more compilers that recognize __restrict but this + * covers the major ones. + */ +# define XXH_RESTRICT __restrict +#else +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#ifndef XXH_HAS_INCLUDE +# ifdef __has_include +/* + * Not defined as XXH_HAS_INCLUDE(x) (function-like) because + * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) + */ +# define XXH_HAS_INCLUDE __has_include +# else +# define XXH_HAS_INCLUDE(x) 0 +# endif +#endif + +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_FEATURE_SVE) +# include <arm_sve.h> +# endif +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || (defined(_M_ARM) && _M_ARM >= 7) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* WASM SIMD128 via SIMDe */ +# define inline __inline__ /* circumvent a clang bug */ +# include <arm_neon.h> +# undef inline +# elif defined(__AVX2__) +# include <immintrin.h> +# elif defined(__SSE2__) +# include <emmintrin.h> +# endif +#endif + +#if defined(_MSC_VER) +# include <intrin.h> +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * internal macro XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< + * NEON for most ARMv7-A, all AArch64, and WASM SIMD128 + * via the SIMDeverywhere polyfill provided with the + * Emscripten SDK. + */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ + XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +# define XXH_SVE 6 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__ARM_FEATURE_SVE) +# define XXH_VECTOR XXH_SVE +# elif ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* wasm simd128 via SIMDe */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ +#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) +# ifdef _MSC_VER +# pragma warning(once : 4606) +# else +# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." +# endif +# undef XXH_VECTOR +# define XXH_VECTOR XXH_SCALAR +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_SVE /* sve */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#elif XXH_VECTOR == XXH_SVE +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +#if defined(__GNUC__) || defined(__clang__) +# define XXH_ALIASING __attribute__((may_alias)) +#else +# define XXH_ALIASING /* nothing */ +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + +#if XXH_VECTOR == XXH_NEON + +/* + * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 + * optimizes out the entire hashLong loop because of the aliasing violation. + * + * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, + * so the only option is to mark it as aliasing. + */ +typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; + +/*! + * @internal + * @brief `vld1q_u64` but faster and alignment-safe. + * + * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only + * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). + * + * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it + * prohibits load-store optimizations. Therefore, a direct dereference is used. + * + * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe + * unaligned load. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ +{ + return *(xxh_aliasing_uint64x2_t const *)ptr; +} +#else +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) +{ + return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); +} +#endif + +/*! + * @internal + * @brief `vmlal_u32` on low and high halves of a vector. + * + * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with + * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` + * with `vmlal_u32`. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* Inline assembly is the only way */ + __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); + return acc; +} +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* This intrinsic works as expected */ + return vmlal_high_u32(acc, lhs, rhs); +} +#else +/* Portable intrinsic versions */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); +} +/*! @copydoc XXH_vmlal_low_u32 + * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); +} +#endif + +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * This can be set to 2, 4, 6, or 8. + * + * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those + * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU + * bandwidth. + * + * This is even more noticeable on the more advanced cores like the Cortex-A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes + * and 2 scalar lanes, which is chosen by default. + * + * This does not apply to Apple processors or 32-bit processors, which run better with + * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * most other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning + * it effectively becomes worse 4. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, + * and `pixel`. This is a problem for obvious reasons. + * + * These keywords are unnecessary; the spec literally says they are + * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd + * after including the header. + * + * We use pragma push_macro/pop_macro to keep the namespace clean. */ +# pragma push_macro("bool") +# pragma push_macro("vector") +# pragma push_macro("pixel") +/* silence potential macro redefined warnings */ +# undef bool +# undef vector +# undef pixel + +# if defined(__s390x__) +# include <s390intrin.h> +# else +# include <altivec.h> +# endif + +/* Restore the original macro values, if applicable. */ +# pragma pop_macro("pixel") +# pragma pop_macro("vector") +# pragma pop_macro("bool") + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +/* + * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. + */ +typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ + /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + +#if XXH_VECTOR == XXH_SVE +#define ACCRND(acc, offset) \ +do { \ + svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ + svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ + svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ + svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ + svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ + svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ + svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ + acc = svadd_u64_x(mask, acc, mul); \ +} while (0) +#endif /* XXH_VECTOR == XXH_SVE */ + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if XXH_SIZE_OPT >= 1 +# define XXH_PREFETCH(ptr) (void)(ptr) +# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + +static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ +static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= PRIME_MX1; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= PRIME_MX2; + h64 ^= (h64 >> 35) + len ; + h64 *= PRIME_MX2; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; +#if XXH_SIZE_OPT >= 1 + /* Smaller and cleaner, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); + acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); + } while (i-- != 0); +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); +#endif + return XXH3_avalanche(acc); + } +} + +/*! + * @brief Maximum size of "short" key in bytes. + */ +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + xxh_u64 acc_end; + unsigned int const nbRounds = (unsigned int)len / 16; + unsigned int i; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + /* last bytes */ + acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + XXH_ASSERT(nbRounds >= 8); + acc = XXH3_avalanche(acc); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + /* + * Prevents clang for unrolling the acc loop and interleaving with this one. + */ + XXH_COMPILER_GUARD(acc); + acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + return XXH3_avalanche(acc + acc_end); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * These macros are to generate an XXH3_accumulate() function. + * The two arguments select the name suffix and target attribute. + * + * The name of this symbol is XXH3_accumulate_<name>() and it calls + * XXH3_accumulate_512_<name>(). + * + * It may be useful to hand implement this function if the compiler fails to + * optimize the inline function. + */ +#define XXH3_ACCUMULATE_TEMPLATE(name) \ +void \ +XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ + const xxh_u8* XXH_RESTRICT input, \ + const xxh_u8* XXH_RESTRICT secret, \ + size_t nbStripes) \ +{ \ + size_t n; \ + for (n = 0; n < nbStripes; n++ ) { \ + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ + XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ + XXH3_accumulate_512_##name( \ + acc, \ + in, \ + secret + n*XXH_SECRET_CONSUME_RATE); \ + } \ +} + + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} +XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); + __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); + +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); + +/*! + * @internal + * @brief The bulk processing loop for NEON and WASM SIMD128. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + * + * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit + * integers instead of the other platforms which mask full 64-bit vectors, + * so the setup is more complicated than just shifting right. + * + * Additionally, there is an optimization for 4 lanes at once noted below. + * + * Since, as stated, the most optimal amount of lanes for Cortexes is 6, + * there needs to be *three* versions of the accumulate operation used + * for the remaining 2 lanes. + * + * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap + * nearly perfectly. + */ + +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { /* GCC for darwin arm64 does not like aliasing here */ + xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* xinput = (const uint8_t *) input; + uint8_t const* xsecret = (const uint8_t *) secret; + + size_t i; +#ifdef __wasm_simd128__ + /* + * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret + * is constant propagated, which results in it converting it to this + * inside the loop: + * + * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) + * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) + * ... + * + * This requires a full 32-bit address immediate (and therefore a 6 byte + * instruction) as well as an add for each offset. + * + * Putting an asm guard prevents it from folding (at the cost of losing + * the alignment hint), and uses the free offset in `v128.load` instead + * of adding secret_offset each time which overall reduces code size by + * about a kilobyte and improves performance. + */ + XXH_COMPILER_GUARD(xsecret); +#endif + /* Scalar lanes use the normal scalarRound routine */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + i = 0; + /* 4 NEON lanes at a time. */ + for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); + uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); + /* data_swap = swap(data_vec) */ + uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); + uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); + uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); + + /* + * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a + * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to + * get one vector with the low 32 bits of each lane, and one vector + * with the high 32 bits of each lane. + * + * The intrinsic returns a double vector because the original ARMv7-a + * instruction modified both arguments in place. AArch64 and SIMD128 emit + * two instructions from this intrinsic. + * + * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] + * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] + */ + uint32x4x2_t unzipped = vuzpq_u32( + vreinterpretq_u32_u64(data_key_1), + vreinterpretq_u32_u64(data_key_2) + ); + /* data_key_lo = data_key & 0xFFFFFFFF */ + uint32x4_t data_key_lo = unzipped.val[0]; + /* data_key_hi = data_key >> 32 */ + uint32x4_t data_key_hi = unzipped.val[1]; + /* + * Then, we can split the vectors horizontally and multiply which, as for most + * widening intrinsics, have a variant that works on both high half vectors + * for free on AArch64. A similar instruction is available on SIMD128. + * + * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi + */ + uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); + uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); + /* + * Clang reorders + * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s + * c += a; // add acc.2d, acc.2d, swap.2d + * to + * c += a; // add acc.2d, acc.2d, swap.2d + * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s + * + * While it would make sense in theory since the addition is faster, + * for reasons likely related to umlal being limited to certain NEON + * pipelines, this is worse. A compiler guard fixes this. + */ + XXH_COMPILER_GUARD_CLANG_NEON(sum_1); + XXH_COMPILER_GUARD_CLANG_NEON(sum_2); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum_1); + xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); + } + /* Operate on the remaining NEON lanes 2 at a time. */ + for (; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + /* acc_vec_2 = swap(data_vec) */ + uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* For two lanes, just use VMOVN and VSHRN. */ + /* data_key_lo = data_key & 0xFFFFFFFF; */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* data_key_hi = data_key >> 32; */ + uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); + /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ + uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); + /* Same Clang workaround as before */ + XXH_COMPILER_GUARD_CLANG_NEON(sum); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64 (xacc[i], sum); + } + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + + size_t i; + /* WASM uses operator overloads and doesn't need these. */ +#ifndef __wasm_simd128__ + /* { prime32_1, prime32_1 } */ + uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); + /* { 0, prime32_1, 0, prime32_1 } */ + uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); +#endif + + /* AArch64 uses both scalar and neon at the same time */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); + uint64x2_t data_vec = veorq_u64(acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* xacc[i] *= XXH_PRIME32_1 */ +#ifdef __wasm_simd128__ + /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ + xacc[i] = data_key * XXH_PRIME32_1; +#else + /* + * Expanded version with portable NEON intrinsics + * + * lo(x) * lo(y) + (hi(x) * lo(y) << 32) + * + * prod_hi = hi(data_key) * lo(prime) << 32 + * + * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector + * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits + * and avoid the shift. + */ + uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); + /* Extract low bits for vmlal_u32 */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); +#endif + } + } +} +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ + xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = xacc[i]; + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + xacc[i] = acc_vec; + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + const xxh_u8* const xsecret = (const xxh_u8*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_SVE) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_sve( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc); + ACCRND(vacc, 0); + svst1_u64(mask, xacc, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } +} + +XXH_FORCE_INLINE void +XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes) +{ + if (nbStripes != 0) { + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc + 0); + do { + /* svprfd(svbool_t, void *, enum svfprop); */ + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(vacc, 0); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } + } +} + +#endif + +/* scalar variants - universal */ + +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +/* + * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they + * emit an excess mask and a full 64-bit multiply-add (MADD X-form). + * + * While this might not seem like much, as AArch64 is a 64-bit architecture, only + * big Cortex designs have a full 64-bit multiplier. + * + * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit + * multiplies expand to 2-3 multiplies in microcode. This has a major penalty + * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. + * + * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does + * not have this penalty and does the mask automatically. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + xxh_u64 ret; + /* note: %x = 64-bit register, %w = 32-bit register */ + __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); + return ret; +} +#else +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; +} +#endif + +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); + } +} + +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ +#if defined(__GNUC__) && !defined(__clang__) \ + && (defined(__arm__) || defined(__thumb2__)) \ + && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ + && XXH_SIZE_OPT <= 0 +# pragma GCC unroll 8 +#endif + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) + +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} + +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__GNUC__) && defined(__aarch64__) + /* + * UGLY HACK: + * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes the compiler to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes the compiler to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_accumulate XXH3_accumulate_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_accumulate XXH3_accumulate_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_accumulate XXH3_accumulate_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_accumulate XXH3_accumulate_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_accumulate XXH3_accumulate_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_SVE) +#define XXH3_accumulate_512 XXH3_accumulate_512_sve +#define XXH3_accumulate XXH3_accumulate_sve +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_accumulate XXH3_accumulate_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + +#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ +# undef XXH3_initCustomSecret +# define XXH3_initCustomSecret XXH3_initCustomSecret_scalar +#endif + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ +#if XXH_SIZE_OPT <= 0 + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); +#endif + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) +{ + return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (length <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); +} + + +/* === XXH3 streaming === */ +#ifndef XXH_NO_STREAM +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup XXH3_family */ +/*! + * @brief Allocate an @ref XXH3_state_t. + * + * @return An allocated pointer of @ref XXH3_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH3_freeState(). + */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup XXH3_family */ +/*! + * @brief Frees an @ref XXH3_state_t. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * + * @return @ref XXH_OK. + * + * @note Must be allocated with XXH3_createState(). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/*! + * @internal + * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). + * + * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. + * + * @param acc Pointer to the 8 accumulator lanes + * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* + * @param nbStripesPerBlock Number of stripes in a block + * @param input Input pointer + * @param nbStripes Number of stripes to process + * @param secret Secret pointer + * @param secretLimit Offset of the last block in @p secret + * @param f_acc Pointer to an XXH3_accumulate implementation + * @param f_scramble Pointer to an XXH3_scrambleAcc implementation + * @return Pointer past the end of @p input after processing + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; + /* Process full blocks */ + if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { + /* Process the initial partial block... */ + size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; + + do { + /* Accumulate and scramble */ + f_acc(acc, input, initialSecret, nbStripesThisIter); + f_scramble(acc, secret + secretLimit); + input += nbStripesThisIter * XXH_STRIPE_LEN; + nbStripes -= nbStripesThisIter; + /* Then continue the loop with the full block size */ + nbStripesThisIter = nbStripesPerBlock; + initialSecret = secret; + } while (nbStripes >= nbStripesPerBlock); + *nbStripesSoFarPtr = 0; + } + /* Process a partial block */ + if (nbStripes > 0) { + f_acc(acc, input, initialSecret, nbStripes); + input += nbStripes * XXH_STRIPE_LEN; + *nbStripesSoFarPtr += nbStripes; + } + /* Return end pointer */ + return input; +} + +#ifndef XXH3_STREAM_USE_STACK +# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; + XXH_memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + input = XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, nbStripes, + secret, state->secretLimit, + f_acc, f_scramble); + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + + } + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + XXH_memcpy(state->acc, acc, sizeof(acc)); +#endif + } + + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + const xxh_u8* lastStripePtr; + + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + /* Consume remaining stripes then point to remaining data in buffer */ + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate, XXH3_scrambleAcc); + lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; + } else { /* bufferedSize < XXH_STRIPE_LEN */ + /* Copy to temp buffer */ + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + lastStripePtr = lastStripe; + } + /* Last stripe */ + XXH3_accumulate_512(acc, + lastStripePtr, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= PRIME_MX2; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + +#if XXH_SIZE_OPT >= 1 + { + /* Smaller, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); + } while (i-- != 0); + } +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); +#endif + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + unsigned i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + /* + * We set as `i` as offset + 32. We do this so that unchanged + * `len` can be used as upper bound. This reaches a sweet spot + * where both x86 and aarch64 get simple agen and good codegen + * for the loop. + */ + for (i = 32; i < 160; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + i - 32, + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + /* + * NB: `i <= len` will duplicate the last 32-bytes if + * len % 32 was zero. This is an unfortunate necessity to keep + * the hash result stable. + */ + for (i=160; i <= len; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + (XXH64_hash_t)0 - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong() is not inlined. + */ +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @p secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + * + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ +#ifndef XXH_NO_STREAM +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_64bits_update(state, input, len); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ +/* 128-bit utility functions */ + +#include <string.h> /* memcmp, memcpy */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * @return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif + + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n<nbSeg16; n++) { + XXH128_hash_t const h128 = XXH128(&scrambler, sizeof(scrambler), n); + XXH3_combine16((char*)secretBuffer + n*16, h128); + } + /* last segment */ + XXH3_combine16((char*)secretBuffer + secretSize - 16, XXH128_hashFromCanonical(&scrambler)); + } + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed) +{ + XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + XXH3_initCustomSecret(secret, seed); + XXH_ASSERT(secretBuffer != NULL); + memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE); +} + + + +/* Pop our optimization override from above */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC pop_options +#endif + +#endif /* XXH_NO_LONG_LONG */ + +#endif /* XXH_NO_XXH3 */ + +/*! + * @} + */ +#endif /* XXH_IMPLEMENTATION */ + + +#if defined (__cplusplus) +} /* extern "C" */ +#endif diff --git a/src/bled/xz_dec_lzma2.c b/src/bled/xz_dec_lzma2.c index a6911db837b..7d936a83141 100644 --- a/src/bled/xz_dec_lzma2.c +++ b/src/bled/xz_dec_lzma2.c @@ -10,7 +10,6 @@ #include "xz_private.h" #include "xz_lzma2.h" -#include <assert.h> /* * Range decoder initialization eats the first five bytes of each LZMA chunk. @@ -623,7 +622,7 @@ static void XZ_FUNC lzma_len(struct xz_dec_lzma2 *s, struct lzma_len_dec *l, uint32_t pos_state) { uint16_t *probs; - uint32_t limit, v; + uint32_t limit; if (!rc_bit(&s->rc, &l->choice)) { probs = l->low[pos_state]; @@ -642,9 +641,7 @@ static void XZ_FUNC lzma_len(struct xz_dec_lzma2 *s, struct lzma_len_dec *l, } } - v = s->lzma.len + rc_bittree(&s->rc, probs, limit) - limit; - assert(v >= s->lzma.len); - s->lzma.len = (v < s->lzma.len) ? 0 : v; + s->lzma.len += rc_bittree(&s->rc, probs, limit) - limit; } /* Decode a match. The distance will be stored in s->lzma.rep0. */ @@ -663,7 +660,6 @@ static void XZ_FUNC lzma_match(struct xz_dec_lzma2 *s, uint32_t pos_state) lzma_len(s, &s->lzma.match_len_dec, pos_state); probs = s->lzma.dist_slot[lzma_get_dist_state(s->lzma.len)]; - // coverity[overflow_const] dist_slot = rc_bittree(&s->rc, probs, DIST_SLOTS) - DIST_SLOTS; if (dist_slot < DIST_MODEL_START) { diff --git a/src/bled/zstd.h b/src/bled/zstd.h index 858d434a0ac..9a7b3f53246 100644 --- a/src/bled/zstd.h +++ b/src/bled/zstd.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -7,19 +7,130 @@ * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. */ +#if defined (__cplusplus) +extern "C" { +#endif #ifndef ZSTD_H_235446 #define ZSTD_H_235446 -/* ====== Dependency ======*/ -#include <limits.h> /* INT_MAX */ +/* ====== Dependencies ======*/ #include <stddef.h> /* size_t */ #include "zstd_config.h" - /* ===== ZSTDLIB_API : control library symbols visibility ===== */ -#define ZSTDLIB_VISIBILITY -#define ZSTDLIB_API ZSTDLIB_VISIBILITY +#ifndef ZSTDLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDLIB_VISIBILITY +# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDLIB_API ZSTDLIB_VISIBLE +#endif + +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) || defined(__IAR_SYSTEMS_ICC__) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + + +/******************************************************************************* + Introduction + + zstd, short for Zstandard, is a fast lossless compression algorithm, targeting + real-time compression scenarios at zlib-level and better compression ratios. + The zstd compression library provides in-memory compression and decompression + functions. + + The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), + which is currently 22. Levels >= 20, labeled `--ultra`, should be used with + caution, as they require more memory. The library also offers negative + compression levels, which extend the range of speed vs. ratio preferences. + The lower the level, the faster the speed (at the cost of compression). + + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit context) + - unbounded multiple steps (described as Streaming compression) + + The compression ratio achievable on small data can be highly improved using + a dictionary. Dictionary compression can be performed in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Bulk-processing + dictionary API) + + Advanced experimental functions can be accessed using + `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. + + Advanced experimental APIs should never be used with a dynamically-linked + library. They are not "stable"; their definitions or signatures may change in + the future. Only static linking is allowed. +*******************************************************************************/ + +/*------ Version ------*/ +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 5 +#define ZSTD_VERSION_RELEASE 7 +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) + +/*! ZSTD_versionNumber() : + * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ +ZSTDLIB_API unsigned ZSTD_versionNumber(void); + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) + +/*! ZSTD_versionString() : + * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ +ZSTDLIB_API const char* ZSTD_versionString(void); + +/* ************************************* + * Default constant + ***************************************/ +#ifndef ZSTD_CLEVEL_DEFAULT +# define ZSTD_CLEVEL_DEFAULT 3 +#endif + +/* ************************************* + * Constants + ***************************************/ /* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ #define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ @@ -30,23 +141,553 @@ #define ZSTD_BLOCKSIZELOG_MAX 17 #define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) + +/*************************************** +* Simple Core API +***************************************/ +/*! ZSTD_compress() : + * Compresses `src` content as a single zstd compressed frame into already allocated `dst`. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*! ZSTD_decompress() : + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * Multiple compressed frames can be decompressed at once with this method. + * The result will be the concatenation of all decompressed frames, back to back. + * `dstCapacity` is an upper bound of originalSize to regenerate. + * First frame's decompressed size can be extracted using ZSTD_getFrameContentSize(). + * If maximum upper bound isn't known, prefer using streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + + +/*====== Decompression helper functions ======*/ + +/*! ZSTD_getFrameContentSize() : requires v1.3.0+ + * `src` should point to the start of a ZSTD encoded frame. + * `srcSize` must be at least as large as the frame header. + * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. + * @return : - decompressed size of `src` frame content, if known + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) + * note 1 : a 0 return value means the frame is valid but "empty". + * note 2 : decompressed size is an optional field, it may not be present (typically in streaming mode). + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can rely on some implicit limit, + * as ZSTD_decompress() only needs an upper bound of decompressed size. + * (For example, data could be necessarily cut into blocks <= 16 KB). + * note 3 : decompressed size is always present when compression is completed using single-pass functions, + * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). + * note 4 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure return value fits within application's authorized limits. + * Each application can set its own limits. + * note 6 : This function replaces ZSTD_getDecompressedSize() */ #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); -ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); +/*! ZSTD_getDecompressedSize() : + * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). + * Both functions work the same way, but ZSTD_getDecompressedSize() blends + * "empty", "unknown" and "error" results to the same return value (0), + * while ZSTD_getFrameContentSize() gives them separate return values. + * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ +ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize") +ZSTDLIB_API +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+ + * `src` should point to the start of a ZSTD frame or skippable frame. + * `srcSize` must be >= first frame size + * @return : the compressed size of the first frame starting at `src`, + * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + * or an error code if input is invalid */ +ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); + + +/*====== Compression helper functions ======*/ + +/*! ZSTD_compressBound() : + * maximum compressed size in worst case single-pass scenario. + * When invoking `ZSTD_compress()`, or any other one-pass compression function, + * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize) + * as it eliminates one potential failure scenario, + * aka not enough room in dst buffer to write the compressed frame. + * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE . + * In which case, ZSTD_compressBound() will return an error code + * which can be tested using ZSTD_isError(). + * + * ZSTD_COMPRESSBOUND() : + * same as ZSTD_compressBound(), but as a macro. + * It can be used to produce constants, which can be useful for static allocation, + * for example to size a static array on stack. + * Will produce constant value 0 if srcSize is too large. + */ +#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U) +#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ + + +/*====== Error helper functions ======*/ +#include "zstd_errors.h" /* list of errors */ +/* ZSTD_isError() : + * Most ZSTD_* functions returning a size_t value can be tested for error, + * using ZSTD_isError(). + * @return 1 if error, 0 otherwise + */ +ZSTDLIB_API unsigned ZSTD_isError(size_t result); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); /* convert a result into an error code, which can be compared to error enum list */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t result); /*!< provides readable string from a function result */ +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */ + + +/*************************************** +* Explicit context +***************************************/ +/*= Compression context + * When compressing many times, + * it is recommended to allocate a compression context just once, + * and reuse it for each successive compression operation. + * This will make the workload easier for system's memory. + * Note : re-using context is just a speed / resource optimization. + * It doesn't change the compression ratio, which remains identical. + * Note 2: For parallel execution in multi-threaded environments, + * use one different context per thread . + */ typedef struct ZSTD_CCtx_s ZSTD_CCtx; +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* compatible with NULL pointer */ + +/*! ZSTD_compressCCtx() : + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. + * Important : in order to mirror `ZSTD_compress()` behavior, + * this function compresses at the requested compression level, + * __ignoring any other advanced parameter__ . + * If any advanced parameter was set using the advanced API, + * they will all be reset. Only @compressionLevel remains. + */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*= Decompression context + * When decompressing many times, + * it is recommended to allocate a context only once, + * and reuse it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution. */ typedef struct ZSTD_DCtx_s ZSTD_DCtx; ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ +/*! ZSTD_decompressDCtx() : + * Same as ZSTD_decompress(), + * requires an allocated ZSTD_DCtx. + * Compatible with sticky parameters (see below). + */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/********************************************* +* Advanced compression API (Requires v1.4.0+) +**********************************************/ + +/* API design : + * Parameters are pushed one by one into an existing context, + * using ZSTD_CCtx_set*() functions. + * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. + * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! + * __They do not apply to one-shot variants such as ZSTD_compressCCtx()__ . + * + * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). + * + * This API supersedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove API entry points from experimental which are redundant with this API. + */ + + +/* Compression strategies, listed from fastest to strongest */ +typedef enum { ZSTD_fast=1, + ZSTD_dfast=2, + ZSTD_greedy=3, + ZSTD_lazy=4, + ZSTD_lazy2=5, + ZSTD_btlazy2=6, + ZSTD_btopt=7, + ZSTD_btultra=8, + ZSTD_btultra2=9 + /* note : new strategies _might_ be added in the future. + Only the order (from fast to strong) is guaranteed */ +} ZSTD_strategy; + +typedef enum { + + /* compression parameters + * Note: When compressing with a ZSTD_CDict these parameters are superseded + * by the parameters used to construct the ZSTD_CDict. + * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ + ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. + * Note that exact compression parameters are dynamically determined, + * depending on both compression level and srcSize (when known). + * Default level is ZSTD_CLEVEL_DEFAULT==3. + * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. + * Note 1 : it's possible to pass a negative compression level. + * Note 2 : setting a level does not automatically set all other compression parameters + * to default. Setting this will however eventually dynamically impact the compression + * parameters which have not been manually set. The manually set + * ones will 'stick'. */ + /* Advanced compression parameters : + * It's possible to pin down compression parameters to some specific values. + * In which case, these values are no longer dynamically selected by the compressor */ + ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. + * This will set a memory budget for streaming decompression, + * with larger values requiring more memory + * and typically compressing more. + * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. + * Special: value 0 means "use default windowLog". + * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT + * requires explicitly allowing such size at streaming decompression stage. */ + ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. + * Resulting memory usage is (1 << (hashLog+2)). + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. + * Larger tables improve compression ratio of strategies <= dFast, + * and improve speed of strategies > dFast. + * Special: value 0 means "use default hashLog". */ + ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. + * Resulting memory usage is (1 << (chainLog+2)). + * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. + * Larger tables result in better and slower compression. + * This parameter is useless for "fast" strategy. + * It's still useful when using "dfast" strategy, + * in which case it defines a secondary probe table. + * Special: value 0 means "use default chainLog". */ + ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. + * More attempts result in better and slower compression. + * This parameter is useless for "fast" and "dFast" strategies. + * Special: value 0 means "use default searchLog". */ + ZSTD_c_minMatch=105, /* Minimum size of searched matches. + * Note that Zstandard can still find matches of smaller size, + * it just tweaks its search algorithm to look for this size and larger. + * Larger values increase compression and decompression speed, but decrease ratio. + * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. + * Note that currently, for all strategies < btopt, effective minimum is 4. + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ + ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. + * For strategies btopt, btultra & btultra2: + * Length of Match considered "good enough" to stop search. + * Larger values make compression stronger, and slower. + * For strategy fast: + * Distance between match sampling. + * Larger values make compression faster, and weaker. + * Special: value 0 means "use default targetLength". */ + ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. + * The higher the value of selected strategy, the more complex it is, + * resulting in stronger and slower compression. + * Special: value 0 means "use default strategy". */ + + ZSTD_c_targetCBlockSize=130, /* v1.5.6+ + * Attempts to fit compressed block size into approximately targetCBlockSize. + * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX. + * Note that it's not a guarantee, just a convergence target (default:0). + * No target when targetCBlockSize == 0. + * This is helpful in low bandwidth streaming environments to improve end-to-end latency, + * when a client can make use of partial documents (a prominent example being Chrome). + * Note: this parameter is stable since v1.5.6. + * It was present as an experimental parameter in earlier versions, + * but it's not recommended using it with earlier library versions + * due to massive performance regressions. + */ + /* LDM mode parameters */ + ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB + * except when expressly set to a different value. + * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and + * compression strategy >= ZSTD_btopt (== compression level 16+) */ + ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashRateLog". */ + + /* frame parameters */ + ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) + * Content size must be known at the beginning of compression. + * This is automatically the case when using ZSTD_compress2(), + * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ + ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ + + /* multi-threading parameters */ + /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). + * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. + * In a situation where it's unknown if the linked library supports multi-threading or not, + * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. + */ + ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. + * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : + * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, + * while compression is performed in parallel, within worker thread(s). + * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : + * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). + * More workers improve speed, but also increase memory usage. + * Default value is `0`, aka "single-threaded mode" : no worker is spawned, + * compression is performed inside Caller's thread, and all invocations are blocking */ + ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. + * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. + * 0 means default, which is dynamically determined based on compression parameters. + * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest. + * The minimum size is automatically and transparently enforced. */ + ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. + * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. + * It helps preserve compression ratio, while each job is compressed in parallel. + * This value is enforced only when nbWorkers >= 1. + * Larger values increase compression ratio, but decrease speed. + * Possible values range from 0 to 9 : + * - 0 means "default" : value will be determined by the library, depending on strategy + * - 1 means "no overlap" + * - 9 means "full overlap", using a full window size. + * Each intermediate rank increases/decreases load size by a factor 2 : + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default + * default value varies between 6 and 9, depending on strategy */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_rsyncable + * ZSTD_c_format + * ZSTD_c_forceMaxWindow + * ZSTD_c_forceAttachDict + * ZSTD_c_literalCompressionMode + * ZSTD_c_srcSizeHint + * ZSTD_c_enableDedicatedDictSearch + * ZSTD_c_stableInBuffer + * ZSTD_c_stableOutBuffer + * ZSTD_c_blockDelimiters + * ZSTD_c_validateSequences + * ZSTD_c_blockSplitterLevel + * ZSTD_c_splitAfterSequences + * ZSTD_c_useRowMatchFinder + * ZSTD_c_prefetchCDictTables + * ZSTD_c_enableSeqProducerFallback + * ZSTD_c_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. + */ + ZSTD_c_experimentalParam1=500, + ZSTD_c_experimentalParam2=10, + ZSTD_c_experimentalParam3=1000, + ZSTD_c_experimentalParam4=1001, + ZSTD_c_experimentalParam5=1002, + /* was ZSTD_c_experimentalParam6=1003; is now ZSTD_c_targetCBlockSize */ + ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam8=1005, + ZSTD_c_experimentalParam9=1006, + ZSTD_c_experimentalParam10=1007, + ZSTD_c_experimentalParam11=1008, + ZSTD_c_experimentalParam12=1009, + ZSTD_c_experimentalParam13=1010, + ZSTD_c_experimentalParam14=1011, + ZSTD_c_experimentalParam15=1012, + ZSTD_c_experimentalParam16=1013, + ZSTD_c_experimentalParam17=1014, + ZSTD_c_experimentalParam18=1015, + ZSTD_c_experimentalParam19=1016, + ZSTD_c_experimentalParam20=1017 +} ZSTD_cParameter; + +typedef struct { + size_t error; + int lowerBound; + int upperBound; +} ZSTD_bounds; + +/*! ZSTD_cParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - lower and upper bounds, both inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); + +/*! ZSTD_CCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_cParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is generally only possible during frame initialization (before starting compression). + * Exception : when using multi-threading mode (nbWorkers >= 1), + * the following parameters can be updated _during_ compression (within same frame): + * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. + * new parameters will be active for next job only (after a flush()). + * @return : an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtx_setPledgedSrcSize() : + * Total input data size to be compressed as a single frame. + * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. + * This value will also be controlled at end of frame, and trigger an error if not respected. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. + * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. + * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. + * Note 2 : pledgedSrcSize is only valid once, for the next frame. + * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. + * Note 3 : Whenever all input data is provided and consumed in a single round, + * for example with ZSTD_compress2(), + * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), + * this value is automatically overridden by srcSize instead. + */ +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); + typedef enum { ZSTD_reset_session_only = 1, ZSTD_reset_parameters = 2, ZSTD_reset_session_and_parameters = 3 } ZSTD_ResetDirective; + +/*! ZSTD_CCtx_reset() : + * There are 2 different things that can be reset, independently or jointly : + * - The session : will stop compressing current frame, and make CCtx ready to start a new one. + * Useful after an error, or to interrupt any ongoing compression. + * Any internal data not yet flushed is cancelled. + * Compression parameters and dictionary remain unchanged. + * They will be used to compress next frame. + * Resetting session never fails. + * - The parameters : changes all parameters back to "default". + * This also removes any reference to any dictionary or external sequence producer. + * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) + * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) + * - Both : similar to resetting the session, followed by resetting parameters. + */ +ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); + +/*! ZSTD_compress2() : + * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * (note that this entry point doesn't even expose a compression level parameter). + * ZSTD_compress2() always starts a new frame. + * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - The function is always blocking, returns when compression is completed. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data, though it is possible it fails for other reasons. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/*********************************************** +* Advanced decompression API (Requires v1.4.0+) +************************************************/ + +/* The advanced API pushes parameters one by one into an existing DCtx context. + * Parameters are sticky, and remain valid for all following frames + * using the same DCtx context. + * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). + * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). + * Therefore, no new decompression function is necessary. + */ + +typedef enum { + + ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which + * the streaming API will refuse to allocate memory buffer + * in order to protect the host from unreasonable memory requirements. + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). + * Special: value 0 means "use default maximum windowLog". */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_d_format + * ZSTD_d_stableOutBuffer + * ZSTD_d_forceIgnoreChecksum + * ZSTD_d_refMultipleDDicts + * ZSTD_d_disableHuffmanAssembly + * ZSTD_d_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly + */ + ZSTD_d_experimentalParam1=1000, + ZSTD_d_experimentalParam2=1001, + ZSTD_d_experimentalParam3=1002, + ZSTD_d_experimentalParam4=1003, + ZSTD_d_experimentalParam5=1004, + ZSTD_d_experimentalParam6=1005 + +} ZSTD_dParameter; + +/*! ZSTD_dParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - both lower and upper bounds, inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); + +/*! ZSTD_DCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_dParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is only possible during frame initialization (before starting decompression). + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); + +/*! ZSTD_DCtx_reset() : + * Return a DCtx to clean state. + * Session and parameters can be reset jointly or separately. + * Parameters can only be reset when no active frame is being decompressed. + * @return : 0, or an error code, which can be tested with ZSTD_isError() + */ ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + +/**************************** +* Streaming +****************************/ + typedef struct ZSTD_inBuffer_s { const void* src; /**< start of input buffer */ size_t size; /**< size of input buffer */ @@ -59,22 +700,543 @@ typedef struct ZSTD_outBuffer_s { size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ } ZSTD_outBuffer; + + +/*-*********************************************************************** +* Streaming compression - HowTo +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* It is recommended to reuse ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. +* +* For parallel execution, use one separate ZSTD_CStream per thread. +* +* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. +* +* Parameters are sticky : when starting a new compression on the same context, +* it will reuse the same sticky parameters as previous compression session. +* When in doubt, it's recommended to fully initialize the context before usage. +* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), +* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to +* set more specific parameters, the pledged source size, or load a dictionary. +* +* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to +* consume input stream. The function will automatically update both `pos` +* fields within `input` and `output`. +* Note that the function may not consume the entire input, for example, because +* the output buffer is already full, in which case `input.pos < input.size`. +* The caller must check if input has been entirely consumed. +* If not, the caller must make some room to receive more compressed data, +* and then present again remaining input data. +* note: ZSTD_e_continue is guaranteed to make some forward progress when called, +* but doesn't guarantee maximal forward progress. This is especially relevant +* when compressing with multiple threads. The call won't block if it can +* consume some input, but if it can't it will wait for some, but not all, +* output to be flushed. +* @return : provides a minimum amount of data remaining to be flushed from internal buffers +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, +* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. +* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). +* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the +* operation. +* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if internal buffers are entirely flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to +* start a new frame. +* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if frame fully completed and fully flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ + /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ +/*===== ZSTD_CStream management functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */ + +/*===== Streaming compression functions =====*/ +typedef enum { + ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ + ZSTD_e_flush=1, /* flush any data provided so far, + * it creates (at least) one new block, that can be decoded immediately on reception; + * frame will continue: any future data can still reference previously compressed data, improving compression. + * note : multithreaded compression will block to flush as much output as possible. */ + ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. + * note that frame is only closed after compressed data is fully flushed (return value == 0). + * After that point, any additional data starts a new frame. + * note : each frame is independent (does not reference any content from previous frame). + : note : multithreaded compression will block to flush as much output as possible. */ +} ZSTD_EndDirective; + +/*! ZSTD_compressStream2() : Requires v1.4.0+ + * Behaves about the same as ZSTD_compressStream, with additional control on end directive. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) + * - output->pos must be <= dstCapacity, input->pos must be <= srcSize + * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - endOp must be a valid directive + * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. + * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, + * and then immediately returns, just indicating that there is some data remaining to be flushed. + * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. + * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. + * - @return provides a minimum amount of data remaining to be flushed from internal buffers + * or an error code, which can be tested using ZSTD_isError(). + * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. + * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. + * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. + * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), + * only ZSTD_e_end or ZSTD_e_flush operations are allowed. + * Before starting a new compression job, or changing compression parameters, + * it is required to fully flush internal buffers. + * - note: if an operation ends with an error, it may leave @cctx in an undefined state. + * Therefore, it's UB to invoke ZSTD_compressStream2() of ZSTD_compressStream() on such a state. + * In order to be re-employed after an error, a state must be reset, + * which can be done explicitly (ZSTD_CCtx_reset()), + * or is sometimes implied by methods starting a new compression job (ZSTD_initCStream(), ZSTD_compressCCtx()) + */ +ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + +/* These buffer sizes are softly recommended. + * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. + * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), + * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. + * + * However, note that these recommendations are from the perspective of a C caller program. + * If the streaming interface is invoked from some other language, + * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, + * a major performance rule is to reduce crossing such interface to an absolute minimum. + * It's not rare that performance ends being spent more into the interface, rather than compression itself. + * In which cases, prefer using large buffers, as large as practical, + * for both input and output, to reduce the nb of roundtrips. + */ +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ + + +/* ***************************************************************************** + * This following is a legacy streaming API, available since v1.0+ . + * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). + * It is redundant, but remains fully supported. + ******************************************************************************/ + +/*! + * Equivalent to: + * + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * + * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API + * to compress with a dictionary. + */ +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +/*! + * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). + * NOTE: The return value is different. ZSTD_compressStream() returns a hint for + * the next read size (if non-zero and not an error). ZSTD_compressStream2() + * returns the minimum nb of bytes left to flush (if non-zero and not an error). + */ +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + + +/*-*************************************************************************** +* Streaming decompression - HowTo +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-employed multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation. +* @return : recommended first input size +* Alternatively, use advanced API to set specific properties. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* +* The function tries to flush all data decoded immediately, respecting output buffer size. +* If `output.pos < output.size`, decoder has flushed everything it could. +* +* However, when `output.pos == output.size`, it's more difficult to know. +* If @return > 0, the frame is not complete, meaning +* either there is still some data left to flush within internal buffers, +* or there is more input to read to complete the frame (or both). +* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. +* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. +* @return : 0 when a frame is completely decoded and fully flushed, +* or an error code, which can be tested using ZSTD_isError(), +* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : +* the return value is a suggested next input size (just a hint for better latency) +* that will never request more than the remaining content of the compressed frame. +* *******************************************************************************/ + typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ + /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ +/*===== ZSTD_DStream management functions =====*/ ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */ +/*===== Streaming decompression functions =====*/ + +/*! ZSTD_initDStream() : + * Initialize/reset DStream state for new decompression operation. + * Call before new decompression operation using same DStream. + * + * Note : This function is redundant with the advanced API and equivalent to: + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, NULL); + */ ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); +/*! ZSTD_decompressStream() : + * Streaming decompression function. + * Call repetitively to consume full input updating it as necessary. + * Function will update both input and output `pos` fields exposing current state via these fields: + * - `input.pos < input.size`, some input remaining and caller should provide remaining input + * on the next call. + * - `output.pos < output.size`, decoder flushed internal output buffer. + * - `output.pos == output.size`, unflushed data potentially present in the internal buffers, + * check ZSTD_decompressStream() @return value, + * if > 0, invoke it again to flush remaining data to output. + * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX. + * + * @return : 0 when a frame is completely decoded and fully flushed, + * or an error code, which can be tested using ZSTD_isError(), + * or any other value > 0, which means there is some decoding or flushing to do to complete current frame. + * + * Note: when an operation returns with an error code, the @zds state may be left in undefined state. + * It's UB to invoke `ZSTD_decompressStream()` on such a state. + * In order to re-use such a state, it must be first reset, + * which can be done explicitly (`ZSTD_DCtx_reset()`), + * or is implied for operations starting some new decompression job (`ZSTD_initDStream`, `ZSTD_decompressDCtx()`, `ZSTD_decompress_usingDict()`) + */ ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + +/************************** +* Simple dictionary API +***************************/ +/*! ZSTD_compress_usingDict() : + * Compression at an explicit compression level using a Dictionary. + * A dictionary can be any arbitrary data segment (also called a prefix), + * or a buffer with specified information (see zdict.h). + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict() : + * Decompression using a known Dictionary. + * Dictionary must be identical to the one used during compression. + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/*********************************** + * Bulk processing dictionary API + **********************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : + * When compressing multiple messages or blocks using the same dictionary, + * it's recommended to digest the dictionary only once, since it's a costly operation. + * ZSTD_createCDict() will create a state from digesting a dictionary. + * The resulting state can be used for future compression operations with very limited startup cost. + * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. + * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, + * in which case the only thing that it transports is the @compressionLevel. + * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, + * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel); + +/*! ZSTD_freeCDict() : + * Function frees memory allocated by ZSTD_createCDict(). + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. + * Note : compression level is _decided at dictionary creation time_, + * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + + typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_freeDDict() : + * Function frees memory allocated with ZSTD_createDDict() + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : + * Decompression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/******************************** + * Dictionary helper functions + *******************************/ + +/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+ + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+ + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+ + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+ + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + +/******************************************************************************* + * Advanced dictionary and prefix API (Requires v1.4.0+) + * + * This API allows dictionaries to be used with ZSTD_compress2(), + * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). + * Dictionaries are sticky, they remain valid when same context is reused, + * they only reset when the context is reset + * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters. + * In contrast, Prefixes are single-use. + ******************************************************************************/ + + +/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal CDict from `dict` buffer. + * Decompression will have to use same dictionary. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames, + * until parameters are reset, a new dictionary is loaded, or the dictionary + * is explicitly invalidated by loading a NULL dictionary. + * Note 2 : Loading a dictionary involves building tables. + * It's also a CPU consuming operation, with non-negligible impact on latency. + * Tables are dependent on compression parameters, and for this reason, + * compression parameters can no longer be changed after loading a dictionary. + * Note 3 :`dict` content will be copied internally. + * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + * In such a case, dictionary buffer must outlive its users. + * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + * to precisely select how dictionary content must be interpreted. + * Note 5 : This method does not benefit from LDM (long distance mode). + * If you want to employ LDM on some large dictionary content, + * prefer employing ZSTD_CCtx_refPrefix() described below. + */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used for all future compressed frames. + * Note that compression parameters are enforced from within CDict, + * and supersede any compression parameter previously set within CCtx. + * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. + * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. + * The dictionary will remain valid for future compressed frames using same CCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Referencing a NULL CDict means "return to no-dictionary mode". + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ +ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) for next compressed frame. + * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + * Decompression will need same prefix to properly regenerate data. + * Compressing with a prefix is similar in outcome as performing a diff and compressing it, + * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * This method is compatible with LDM (long distance mode). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + * Note 1 : Prefix buffer is referenced. It **must** outlive compression. + * Its content must remain unmodified during compression. + * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + * ensure that the window size is large enough to contain the entire source. + * See ZSTD_c_windowLog. + * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + * It's a CPU consuming operation, with non-negligible impact on latency. + * If there is a need to use the same prefix multiple times, consider loadDictionary instead. + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). + * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, + const void* prefix, size_t prefixSize); + +/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal DDict from dict buffer, to be used to decompress all future frames. + * The dictionary remains valid for all future frames, until explicitly invalidated, or + * a new dictionary is loaded. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Loading a dictionary involves building tables, + * which has a non-negligible impact on CPU usage and latency. + * It's recommended to "load once, use many times", to amortize the cost + * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. + * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. + * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of + * how dictionary content is loaded and interpreted. + */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used to decompress next frames. + * The dictionary remains active for decompression of future frames using same DCtx. + * + * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function + * will store the DDict references in a table, and the DDict used for decompression + * will be determined at decompression time, as per the dict ID in the frame. + * The memory for the table is allocated on the first call to refDDict, and can be + * freed with ZSTD_freeDCtx(). + * + * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary + * will be managed, and referencing a dictionary effectively "discards" any previous one. + * + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: referencing a NULL DDict means "return to no-dictionary mode". + * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) to decompress next frame. + * This is the reverse operation of ZSTD_CCtx_refPrefix(), + * and must use the same prefix as the one used during compression. + * Prefix is **only used once**. Reference is discarded at end of frame. + * End of frame is reached when ZSTD_decompressStream() returns 0. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary + * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. + * Prefix buffer must remain unmodified up to the end of frame, + * reached when ZSTD_decompressStream() returns 0. + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). + * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) + * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. + * A full dictionary is more costly, as it requires building tables. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, + const void* prefix, size_t prefixSize); + +/* === Memory management === */ + +/*! ZSTD_sizeof_*() : Requires v1.4.0+ + * These functions give the _current_ memory usage of selected object. + * Note that object memory usage can evolve (increase or decrease) over time. */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + +#endif /* ZSTD_H_235446 */ + + +/* ************************************************************************************** + * ADVANCED AND EXPERIMENTAL FUNCTIONS + **************************************************************************************** + * The definitions in the following section are considered experimental. + * They are provided for advanced scenarios. + * They should never be used with a dynamic library, as prototypes may change in the future. + * Use them only in association with static linking. + * ***************************************************************************************/ + +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY + +#include <limits.h> /* INT_MAX */ + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZSTDLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE +# else +# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE +# endif +#endif + +/**************************************************************************************** + * experimental API (static linking only) + **************************************************************************************** + * The following symbols and constants + * are not planned to join "stable API" status in the near future. + * They can still change in future versions. + * Some of them are planned to remain in the static_only section indefinitely. + * Some of them might be removed in the future (especially when redundant with existing stable functions) + * ***************************************************************************************/ #define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ #define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) @@ -85,13 +1247,124 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); #define ZSTD_WINDOWLOG_MAX_32 30 #define ZSTD_WINDOWLOG_MAX_64 31 #define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX_32 29 +#define ZSTD_CHAINLOG_MAX_64 30 +#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ +#define ZSTD_STRATEGY_MIN ZSTD_fast +#define ZSTD_STRATEGY_MAX ZSTD_btultra2 +#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */ + +#define ZSTD_OVERLAPLOG_MIN 0 +#define ZSTD_OVERLAPLOG_MAX 9 #define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame - */ + * requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size, + * to preserve host's memory from unreasonable requirements. + * This limit can be overridden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,). + * The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */ + + +/* LDM parameter bounds */ +#define ZSTD_LDM_HASHLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_LDM_HASHLOG_MAX ZSTD_HASHLOG_MAX +#define ZSTD_LDM_MINMATCH_MIN 4 +#define ZSTD_LDM_MINMATCH_MAX 4096 +#define ZSTD_LDM_BUCKETSIZELOG_MIN 1 +#define ZSTD_LDM_BUCKETSIZELOG_MAX 8 +#define ZSTD_LDM_HASHRATELOG_MIN 0 +#define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) +/* Advanced parameter bounds */ +#define ZSTD_TARGETCBLOCKSIZE_MIN 1340 /* suitable to fit into an ethernet / wifi / 4G transport frame */ +#define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_SRCSIZEHINT_MIN 0 +#define ZSTD_SRCSIZEHINT_MAX INT_MAX +/* --- Advanced types --- */ + +typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; + +typedef struct { + unsigned int offset; /* The offset of the match. (NOT the same as the offset code) + * If offset == 0 and matchLength == 0, this sequence represents the last + * literals in the block of litLength size. + */ + + unsigned int litLength; /* Literal length of the sequence. */ + unsigned int matchLength; /* Match length of the sequence. */ + + /* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0. + * In this case, we will treat the sequence as a marker for a block boundary. + */ + + unsigned int rep; /* Represents which repeat offset is represented by the field 'offset'. + * Ranges from [0, 3]. + * + * Repeat offsets are essentially previous offsets from previous sequences sorted in + * recency order. For more detail, see doc/zstd_compression_format.md + * + * If rep == 0, then 'offset' does not contain a repeat offset. + * If rep > 0: + * If litLength != 0: + * rep == 1 --> offset == repeat_offset_1 + * rep == 2 --> offset == repeat_offset_2 + * rep == 3 --> offset == repeat_offset_3 + * If litLength == 0: + * rep == 1 --> offset == repeat_offset_2 + * rep == 2 --> offset == repeat_offset_3 + * rep == 3 --> offset == repeat_offset_1 - 1 + * + * Note: This field is optional. ZSTD_generateSequences() will calculate the value of + * 'rep', but repeat offsets do not necessarily need to be calculated from an external + * sequence provider's perspective. For example, ZSTD_compressSequences() does not + * use this 'rep' field at all (as of now). + */ +} ZSTD_Sequence; + +typedef struct { + unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /**< dispatch table : larger == faster, more memory */ + unsigned searchLog; /**< nb of searches : larger == more compression, slower */ + unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ + ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ +} ZSTD_compressionParameters; + +typedef struct { + int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ + int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ + int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ +} ZSTD_frameParameters; + +typedef struct { + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; +} ZSTD_parameters; + +typedef enum { + ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ + ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ + ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ +} ZSTD_dictContentType_e; + +typedef enum { + ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ + ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ +} ZSTD_dictLoadMethod_e; + typedef enum { ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. @@ -111,12 +1384,112 @@ typedef enum { ZSTD_rmd_refMultipleDDicts = 1 } ZSTD_refMultipleDDicts_e; -ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); +typedef enum { + /* Note: this enum and the behavior it controls are effectively internal + * implementation details of the compressor. They are expected to continue + * to evolve and should be considered only in the context of extremely + * advanced performance tuning. + * + * Zstd currently supports the use of a CDict in three ways: + * + * - The contents of the CDict can be copied into the working context. This + * means that the compression can search both the dictionary and input + * while operating on a single set of internal tables. This makes + * the compression faster per-byte of input. However, the initial copy of + * the CDict's tables incurs a fixed cost at the beginning of the + * compression. For small compressions (< 8 KB), that copy can dominate + * the cost of the compression. + * + * - The CDict's tables can be used in-place. In this model, compression is + * slower per input byte, because the compressor has to search two sets of + * tables. However, this model incurs no start-up cost (as long as the + * working context's tables can be reused). For small inputs, this can be + * faster than copying the CDict's tables. + * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. + * + * Zstd has a simple internal heuristic that selects which strategy to use + * at the beginning of a compression. However, if experimentation shows that + * Zstd is making poor choices, it is possible to override that choice with + * this enum. + */ + ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ + ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ + ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ +} ZSTD_dictAttachPref_e; + +typedef enum { + ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. + * Negative compression levels will be uncompressed, and positive compression + * levels will be compressed. */ + ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be + * emitted if Huffman compression is not profitable. */ + ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ +} ZSTD_literalCompressionMode_e; + +typedef enum { + /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final + * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable + * or ZSTD_ps_disable allow for a force enable/disable the feature. + */ + ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */ + ZSTD_ps_enable = 1, /* Force-enable the feature */ + ZSTD_ps_disable = 2 /* Do not use the feature */ +} ZSTD_paramSwitch_e; -ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); -ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); +/*************************************** +* Frame header and size functions +***************************************/ + +/*! ZSTD_findDecompressedSize() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - decompressed size of all data in all successive frames + * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * note 2 : decompressed size is always present when compression is done with ZSTD_compress() + * note 3 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure result fits within application's authorized limits. + * Each application can set its own limits. + * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to + * read each contained frame header. This is fast as most of the data is skipped, + * however it does mean that all frame data must be present and valid. */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_decompressBound() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - upper-bound for the decompressed size of all data in all successive frames + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. + * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. + * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. + * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: + * upper-bound = # blocks * min(128 KB, Window_Size) + */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); + +/*! ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); -/*===== Buffer-less streaming decompression functions =====*/ typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; typedef struct { unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ @@ -126,23 +1499,1673 @@ typedef struct { unsigned headerSize; unsigned dictID; unsigned checksumFlag; + unsigned _reserved1; + unsigned _reserved2; } ZSTD_frameHeader; -ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ -ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); -ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ +/*! ZSTD_getFrameHeader() : + * decode Frame Header, or requires larger `srcSize`. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); + +/*! ZSTD_decompressionMargin() : + * Zstd supports in-place decompression, where the input and output buffers overlap. + * In this case, the output buffer must be at least (Margin + Output_Size) bytes large, + * and the input buffer must be at the end of the output buffer. + * + * _______________________ Output Buffer ________________________ + * | | + * | ____ Input Buffer ____| + * | | | + * v v v + * |---------------------------------------|-----------|----------| + * ^ ^ ^ + * |___________________ Output_Size ___________________|_ Margin _| + * + * NOTE: See also ZSTD_DECOMPRESSION_MARGIN(). + * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or + * ZSTD_decompressDCtx(). + * NOTE: This function supports multi-frame input. + * + * @param src The compressed frame(s) + * @param srcSize The size of the compressed frame(s) + * @returns The decompression margin or an error that can be checked with ZSTD_isError(). + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize); + +/*! ZSTD_DECOMPRESS_MARGIN() : + * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from + * the compressed frame, compute it from the original size and the blockSizeLog. + * See ZSTD_decompressionMargin() for details. + * + * WARNING: This macro does not support multi-frame input, the input must be a single + * zstd frame. If you need that support use the function, or implement it yourself. + * + * @param originalSize The original uncompressed size of the data. + * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX). + * Unless you explicitly set the windowLog smaller than + * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX. + */ +#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \ + ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \ + 4 /* checksum */ + \ + ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \ + (blockSize) /* One block of margin */ \ + )) -ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); +typedef enum { + ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ + ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ +} ZSTD_sequenceFormat_e; -ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +/*! ZSTD_sequenceBound() : + * `srcSize` : size of the input buffer + * @return : upper-bound for the number of sequences that can be generated + * from a buffer of srcSize bytes + * + * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence). + */ +ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize); -ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); -typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; -ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); +/*! ZSTD_generateSequences() : + * WARNING: This function is meant for debugging and informational purposes ONLY! + * Its implementation is flawed, and it will be deleted in a future version. + * It is not guaranteed to succeed, as there are several cases where it will give + * up and fail. You should NOT use this function in production code. + * + * This function is deprecated, and will be removed in a future version. + * + * Generate sequences using ZSTD_compress2(), given a source buffer. + * + * @param zc The compression context to be used for ZSTD_compress2(). Set any + * compression parameters you need on this context. + * @param outSeqs The output sequences buffer of size @p outSeqsSize + * @param outSeqsSize The size of the output sequences buffer. + * ZSTD_sequenceBound(srcSize) is an upper bound on the number + * of sequences that can be generated. + * @param src The source buffer to generate sequences from of size @p srcSize. + * @param srcSize The size of the source buffer. + * + * Each block will end with a dummy sequence + * with offset == 0, matchLength == 0, and litLength == length of last literals. + * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) + * simply acts as a block delimiter. + * + * @returns The number of sequences generated, necessarily less than + * ZSTD_sequenceBound(srcSize), or an error code that can be checked + * with ZSTD_isError(). + */ +ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()") +ZSTDLIB_STATIC_API size_t +ZSTD_generateSequences(ZSTD_CCtx* zc, + ZSTD_Sequence* outSeqs, size_t outSeqsSize, + const void* src, size_t srcSize); + +/*! ZSTD_mergeBlockDelimiters() : + * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals + * by merging them into the literals of the next sequence. + * + * As such, the final generated result has no explicit representation of block boundaries, + * and the final last literals segment is not represented in the sequences. + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters + * @return : number of sequences left after merging + */ +ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); + +/*! ZSTD_compressSequences() : + * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst. + * @src contains the entire input (not just the literals). + * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals + * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) + * The entire source is compressed into a single frame. + * + * The compression behavior changes based on cctx params. In particular: + * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on + * the block size derived from the cctx, and sequences may be split. This is the default setting. + * + * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * + * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined + * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. + * + * In addition to the two adjustable experimental params, there are other important cctx params. + * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. + * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. + * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset + * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md + * + * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. + * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, + * and cannot emit an RLE block that disagrees with the repcode history + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); + + +/*! ZSTD_writeSkippableFrame() : + * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number, + * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so + * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * + * Returns an error if destination buffer is not large enough, if the source size is not representable + * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant); + +/*! ZSTD_readSkippableFrame() : + * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant, + const void* src, size_t srcSize); + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + */ +ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); + + + +/*************************************** +* Memory management +***************************************/ + +/*! ZSTD_estimate*() : + * These functions make it possible to estimate memory usage + * of a future {D,C}Ctx, before its creation. + * This is useful in combination with ZSTD_initStatic(), + * which makes it possible to employ a static buffer for ZSTD_CCtx* state. + * + * ZSTD_estimateCCtxSize() will provide a memory budget large enough + * to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2() + * associated with any compression level up to max specified one. + * The estimate will assume the input may be arbitrarily large, + * which is the worst case. + * + * Note that the size estimation is specific for one-shot compression, + * it is not valid for streaming (see ZSTD_estimateCStreamSize*()) + * nor other potential ways of using a ZSTD_CCtx* state. + * + * When srcSize can be bound by a known and rather "small" value, + * this knowledge can be used to provide a tighter budget estimation + * because the ZSTD_CCtx* state will need less memory for small inputs. + * This tighter estimation can be provided by employing more advanced functions + * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), + * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). + * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. + * + * Note : only single-threaded compression is supported. + * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void); + +/*! ZSTD_estimateCStreamSize() : + * ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression + * using any compression level up to the max specified one. + * It will also consider src size to be arbitrarily "large", which is a worst case scenario. + * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note : CStream size estimation is only correct for single-threaded compression. + * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + * + * ZSTD_DStream memory budget depends on frame's window Size. + * This information can be passed manually, using ZSTD_estimateDStreamSize, + * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); + * Any frame requesting a window size larger than max specified one will be rejected. + * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), + * an internal ?Dict will be created, which additional size is not estimated here. + * In this case, get total size by adding ZSTD_estimate?DictSize + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); + +/*! ZSTD_estimate?DictSize() : + * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). + * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). + * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); + +/*! ZSTD_initStatic*() : + * Initialize an object using a pre-allocated fixed-size buffer. + * workspace: The memory area to emplace the object into. + * Provided pointer *must be 8-bytes aligned*. + * Buffer must outlive object. + * workspaceSize: Use ZSTD_estimate*Size() to determine + * how large workspace must be to support target scenario. + * @return : pointer to object (same address as workspace, just different type), + * or NULL if error (size too small, incorrect alignment, etc.) + * Note : zstd will never resize nor malloc() when using a static buffer. + * If the object requires more memory than available, + * zstd will just error out (typically ZSTD_error_memory_allocation). + * Note 2 : there is no corresponding "free" function. + * Since workspace is allocated externally, it must be freed externally too. + * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level + * into its associated cParams. + * Limitation 1 : currently not compatible with internal dictionary creation, triggered by + * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). + * Limitation 2 : static cctx currently not compatible with multi-threading. + * Limitation 3 : static dctx is incompatible with legacy support. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ + +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ + +ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams); + +ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType); + + +/*! Custom memory allocation : + * These prototypes make it possible to pass your own allocation/free functions. + * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + * All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones. + */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif + +#if defined(__clang__) && __clang_major__ >= 5 +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" +#endif +ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ +#if defined(__clang__) && __clang_major__ >= 5 +#pragma clang diagnostic pop +#endif + +MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); +} + +MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + ZSTD_memset(ptr, 0, size); + return ptr; + } + return ZSTD_calloc(1, size); +} + +MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +{ + if (ptr != NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + ZSTD_free(ptr); + } +} + +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem); + +/*! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer. + */ +typedef struct POOL_ctx_s ZSTD_threadPool; +ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); + + +/* + * This API is temporary and is expected to change or disappear in the future! + */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* cctxParams, + ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem); + + +/*************************************** +* Advanced compression functions +***************************************/ + +/*! ZSTD_createCDict_byReference() : + * Create a digested dictionary for compression + * Dictionary content is just referenced, not duplicated. + * As a consequence, `dictBuffer` **must** outlive CDict, + * and its content must remain unmodified throughout the lifetime of CDict. + * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + * `estimatedSrcSize` value is optional, select 0 if not known */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_getParams() : + * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. + * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ +ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_checkCParams() : + * Ensure param values remain within authorized range. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ +ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustCParams() : + * optimize params for a given `srcSize` and `dictSize`. + * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. + * `dictSize` must be `0` when there is no dictionary. + * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. + * This function never fails (wide contract) */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_CCtx_setCParams() : + * Set all parameters provided within @p cparams into the working @p cctx. + * Note : if modifying parameters during compression (MT mode only), + * note that changes to the .windowLog parameter will be ignored. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + * On failure, no parameters are updated. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams); + +/*! ZSTD_CCtx_setFParams() : + * Set all parameters provided within @p fparams into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams); + +/*! ZSTD_CCtx_setParams() : + * Set all parameters provided within @p params into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params); + +/*! ZSTD_compress_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + +/*! ZSTD_compress_usingCDict_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams); + + +/*! ZSTD_CCtx_loadDictionary_byReference() : + * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. + * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_loadDictionary_advanced() : + * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_CCtx_refPrefix_advanced() : + * Same as ZSTD_CCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/* === experimental parameters === */ +/* these parameters can be used with ZSTD_setParameter() + * they are not guaranteed to remain supported in the future */ + + /* Enables rsyncable mode, + * which makes compressed files more rsync friendly + * by adding periodic synchronization points to the compressed data. + * The target average block size is ZSTD_c_jobSize / 2. + * It's possible to modify the job size to increase or decrease + * the granularity of the synchronization point. + * Once the jobSize is smaller than the window size, + * it will result in compression ratio degradation. + * NOTE 1: rsyncable mode only works when multithreading is enabled. + * NOTE 2: rsyncable performs poorly in combination with long range mode, + * since it will decrease the effectiveness of synchronization points, + * though mileage may vary. + * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. + * If the selected compression level is already running significantly slower, + * the overall speed won't be significantly impacted. + */ + #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 + +/* Select a compression format. + * The value must be of type ZSTD_format_e. + * See ZSTD_format_e enum definition for details */ +#define ZSTD_c_format ZSTD_c_experimentalParam2 + +/* Force back-reference distances to remain < windowSize, + * even when referencing into Dictionary content (default:0) */ +#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 + +/* Controls whether the contents of a CDict + * are used in place, or copied into the working context. + * Accepts values from the ZSTD_dictAttachPref_e enum. + * See the comments on that enum for an explanation of the feature. */ +#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 + +/* Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never compress literals. + * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals + * may still be emitted if huffman is not beneficial to use.) + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * literals compression based on the compression parameters - specifically, + * negative compression levels do not use literal compression. + */ +#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 + +/* User's best guess of source size. + * Hint is not valid when srcSizeHint == 0. + * There is no guarantee that hint is close to actual source size, + * but compression ratio may regress significantly if guess considerably underestimates */ +#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 + +/* Controls whether the new and experimental "dedicated dictionary search + * structure" can be used. This feature is still rough around the edges, be + * prepared for surprising behavior! + * + * How to use it: + * + * When using a CDict, whether to use this feature or not is controlled at + * CDict creation, and it must be set in a CCtxParams set passed into that + * construction (via ZSTD_createCDict_advanced2()). A compression will then + * use the feature or not based on how the CDict was constructed; the value of + * this param, set in the CCtx, will have no effect. + * + * However, when a dictionary buffer is passed into a CCtx, such as via + * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control + * whether the CDict that is created internally can use the feature or not. + * + * What it does: + * + * Normally, the internal data structures of the CDict are analogous to what + * would be stored in a CCtx after compressing the contents of a dictionary. + * To an approximation, a compression using a dictionary can then use those + * data structures to simply continue what is effectively a streaming + * compression where the simulated compression of the dictionary left off. + * Which is to say, the search structures in the CDict are normally the same + * format as in the CCtx. + * + * It is possible to do better, since the CDict is not like a CCtx: the search + * structures are written once during CDict creation, and then are only read + * after that, while the search structures in the CCtx are both read and + * written as the compression goes along. This means we can choose a search + * structure for the dictionary that is read-optimized. + * + * This feature enables the use of that different structure. + * + * Note that some of the members of the ZSTD_compressionParameters struct have + * different semantics and constraints in the dedicated search structure. It is + * highly recommended that you simply set a compression level in the CCtxParams + * you pass into the CDict creation call, and avoid messing with the cParams + * directly. + * + * Effects: + * + * This will only have any effect when the selected ZSTD_strategy + * implementation supports this feature. Currently, that's limited to + * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. + * + * Note that this means that the CDict tables can no longer be copied into the + * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be + * usable. The dictionary can only be attached or reloaded. + * + * In general, you should expect compression to be faster--sometimes very much + * so--and CDict creation to be slightly slower. Eventually, we will probably + * make this mode the default. + */ +#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 + +/* ZSTD_c_stableInBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the compressor that input data presented with ZSTD_inBuffer + * will ALWAYS be the same between calls. + * Technically, the @src pointer must never be changed, + * and the @pos field can only be updated by zstd. + * However, it's possible to increase the @size field, + * allowing scenarios where more data can be appended after compressions starts. + * These conditions are checked by the compressor, + * and compression will fail if they are not respected. + * Also, data in the ZSTD_inBuffer within the range [src, src + pos) + * MUST not be modified during compression or it will result in data corruption. + * + * When this flag is enabled zstd won't allocate an input window buffer, + * because the user guarantees it can reference the ZSTD_inBuffer until + * the frame is complete. But, it will still allocate an output buffer + * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also + * avoid the memcpy() from the input buffer to the input window buffer. + * + * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, compression WILL fail if conditions are not respected. + * + * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST + * not be modified during compression or it will result in data corruption. + * This is because zstd needs to reference data in the ZSTD_inBuffer to find + * matches. Normally zstd maintains its own window buffer for this purpose, + * but passing this flag tells zstd to rely on user provided buffer instead. + */ +#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 + +/* ZSTD_c_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells he compressor that the ZSTD_outBuffer will not be resized between + * calls. Specifically: (out.size - out.pos) will never grow. This gives the + * compressor the freedom to say: If the compressed data doesn't fit in the + * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to + * always decompress directly into the output buffer, instead of decompressing + * into an internal buffer and copying to the output buffer. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer. It will still allocate the + * input window buffer (see ZSTD_c_stableInBuffer). + * + * Zstd will check that (out.size - out.pos) never grows and return an error + * if it does. While not strictly necessary, this should prevent surprises. + */ +#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 + +/* ZSTD_c_blockDelimiters + * Default is 0 == ZSTD_sf_noBlockDelimiters. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * + * Designates whether or not the given array of ZSTD_Sequence contains block delimiters + * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. + * See the definition of ZSTD_Sequence for more specifics. + */ +#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 + +/* ZSTD_c_validateSequences + * Default is 0 == disabled. Set to 1 to enable sequence validation. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * Designates whether or not we validate sequences provided to ZSTD_compressSequences() + * during function execution. + * + * Without validation, providing a sequence that does not conform to the zstd spec will cause + * undefined behavior, and may produce a corrupted block. + * + * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and + * return an error. + * + */ +#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 + +/* ZSTD_c_blockSplitterLevel + * note: this parameter only influences the first splitter stage, + * which is active before producing the sequences. + * ZSTD_c_splitAfterSequences controls the next splitter stage, + * which is active after sequence production. + * Note that both can be combined. + * Allowed values are between 0 and ZSTD_BLOCKSPLITTER_LEVEL_MAX included. + * 0 means "auto", which will select a value depending on current ZSTD_c_strategy. + * 1 means no splitting. + * Then, values from 2 to 6 are sorted in increasing cpu load order. + * + * Note that currently the first block is never split, + * to ensure expansion guarantees in presence of incompressible data. + */ +#define ZSTD_BLOCKSPLITTER_LEVEL_MAX 6 +#define ZSTD_c_blockSplitterLevel ZSTD_c_experimentalParam20 + +/* ZSTD_c_splitAfterSequences + * This is a stronger splitter algorithm, + * based on actual sequences previously produced by the selected parser. + * It's also slower, and as a consequence, mostly used for high compression levels. + * While the post-splitter does overlap with the pre-splitter, + * both can nonetheless be combined, + * notably with ZSTD_c_blockSplitterLevel at ZSTD_BLOCKSPLITTER_LEVEL_MAX, + * resulting in higher compression ratio than just one of them. + * + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use block splitter. + * Set to ZSTD_ps_enable to always use block splitter. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * block splitting based on the compression parameters. + */ +#define ZSTD_c_splitAfterSequences ZSTD_c_experimentalParam13 + +/* ZSTD_c_useRowMatchFinder + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use row-based matchfinder. + * Set to ZSTD_ps_enable to force usage of row-based matchfinder. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * the row-based matchfinder based on support for SIMD instructions and the window log. + * Note that this only pertains to compression strategies: greedy, lazy, and lazy2 + */ +#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 + +/* ZSTD_c_deterministicRefPrefix + * Default is 0 == disabled. Set to 1 to enable. + * + * Zstd produces different results for prefix compression when the prefix is + * directly adjacent to the data about to be compressed vs. when it isn't. + * This is because zstd detects that the two buffers are contiguous and it can + * use a more efficient match finding algorithm. However, this produces different + * results than when the two buffers are non-contiguous. This flag forces zstd + * to always load the prefix in non-contiguous mode, even if it happens to be + * adjacent to the data, to guarantee determinism. + * + * If you really care about determinism when using a dictionary or prefix, + * like when doing delta compression, you should select this option. It comes + * at a speed penalty of about ~2.5% if the dictionary and data happened to be + * contiguous, and is free if they weren't contiguous. We don't expect that + * intentionally making the dictionary and data contiguous will be worth the + * cost to memcpy() the data. + */ +#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 + +/* ZSTD_c_prefetchCDictTables + * Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto. + * + * In some situations, zstd uses CDict tables in-place rather than copying them + * into the working context. (See docs on ZSTD_dictAttachPref_e above for details). + * In such situations, compression speed is seriously impacted when CDict tables are + * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables + * when they are used in-place. + * + * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit. + * For sufficiently large inputs, zstd will by default memcpy() CDict tables + * into the working context, so there is no need to prefetch. This parameter is + * targeted at a middle range of input sizes, where a prefetch is cheap enough to be + * useful but memcpy() is too expensive. The exact range of input sizes where this + * makes sense is best determined by careful experimentation. + * + * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable, + * but in the future zstd may conditionally enable this feature via an auto-detection + * heuristic for cold CDicts. + * Use ZSTD_ps_disable to opt out of prefetching under any circumstances. + */ +#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16 + +/* ZSTD_c_enableSeqProducerFallback + * Allowed values are 0 (disable) and 1 (enable). The default setting is 0. + * + * Controls whether zstd will fall back to an internal sequence producer if an + * external sequence producer is registered and returns an error code. This fallback + * is block-by-block: the internal sequence producer will only be called for blocks + * where the external sequence producer returns an error code. Fallback parsing will + * follow any other cParam settings, such as compression level, the same as in a + * normal (fully-internal) compression operation. + * + * The user is strongly encouraged to read the full Block-Level Sequence Producer API + * documentation (below) before setting this parameter. */ +#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17 + +/* ZSTD_c_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * This parameter can be used to set an upper bound on the blocksize + * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper + * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make + * compressBound() inaccurate). Only currently meant to be used for testing. + */ +#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18 + +/* ZSTD_c_searchForExternalRepcodes + * This parameter affects how zstd parses external sequences, such as sequences + * provided through the compressSequences() API or from an external block-level + * sequence producer. + * + * If set to ZSTD_ps_enable, the library will check for repeated offsets in + * external sequences, even if those repcodes are not explicitly indicated in + * the "rep" field. Note that this is the only way to exploit repcode matches + * while using compressSequences() or an external sequence producer, since zstd + * currently ignores the "rep" field of external sequences. + * + * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in + * external sequences, regardless of whether the "rep" field has been set. This + * reduces sequence compression overhead by about 25% while sacrificing some + * compression ratio. + * + * The default value is ZSTD_ps_auto, for which the library will enable/disable + * based on compression level. + * + * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is + * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future. + */ +#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19 + + +/*! ZSTD_CCtx_getParameter() : + * Get the requested compression parameter value, selected by enum ZSTD_cParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); + + +/*! ZSTD_CCtx_params : + * Quick howto : + * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure + * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into + * an existing ZSTD_CCtx_params structure. + * This is similar to + * ZSTD_CCtx_setParameter(). + * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to + * an existing CCtx. + * These parameters will be applied to + * all subsequent frames. + * - ZSTD_compressStream2() : Do compression using the CCtx. + * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer. + * + * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() + * for static allocation of CCtx for single-threaded compression. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */ + +/*! ZSTD_CCtxParams_reset() : + * Reset params to default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_init() : + * Initializes the compression parameters of cctxParams according to + * compression level. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); + +/*! ZSTD_CCtxParams_init_advanced() : + * Initializes the compression and frame parameters of cctxParams according to + * params. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); + +/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+ + * Similar to ZSTD_CCtx_setParameter. + * Set one compression parameter, selected by enum ZSTD_cParameter. + * Parameters must be applied to a ZSTD_CCtx using + * ZSTD_CCtx_setParametersUsingCCtxParams(). + * @result : a code representing success or failure (which can be tested with + * ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtxParams_getParameter() : + * Similar to ZSTD_CCtx_getParameter. + * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); + +/*! ZSTD_CCtx_setParametersUsingCCtxParams() : + * Apply a set of ZSTD_CCtx_params to the compression context. + * This can be done even after compression is started, + * if nbWorkers==0, this will have no impact until a new compression is started. + * if nbWorkers>=1, new parameters will be picked up at next job, + * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); + +/*! ZSTD_compressStream2_simpleArgs() : + * Same as ZSTD_compressStream2(), + * but using only integral types as arguments. + * This variant might be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp); + + +/*************************************** +* Advanced decompression functions +***************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size); + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * Dictionary content is referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives DDict, + * it must remain read accessible throughout the lifetime of DDict */ +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_byReference() : + * Same as ZSTD_DCtx_loadDictionary(), + * but references `dict` content instead of copying it into `dctx`. + * This saves memory if `dict` remains around., + * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_advanced() : + * Same as ZSTD_DCtx_loadDictionary(), + * but gives direct control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_refPrefix_advanced() : + * Same as ZSTD_DCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_setMaxWindowSize() : + * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. + * This protects a decoder context from reserving too much memory for itself (potential attack scenario). + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); + +/*! ZSTD_DCtx_getParameter() : + * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); + +/* ZSTD_d_format + * experimental parameter, + * allowing selection between ZSTD_format_e input compression formats + */ +#define ZSTD_d_format ZSTD_d_experimentalParam1 +/* ZSTD_d_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the decompressor, and + * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer + * MUST be large enough to fit the entire decompressed frame. This will be + * checked when the frame content size is known. The data in the ZSTD_outBuffer + * in the range [dst, dst + pos) MUST not be modified during decompression + * or you will get data corruption. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer, but it will still allocate + * an input buffer large enough to fit any compressed block. This will also + * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. + * If you need to avoid the input buffer allocation use the buffer-less + * streaming API. + * + * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, decompression WILL fail if you violate the preconditions. + * + * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST + * not be modified during decompression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate + * matches. Normally zstd maintains its own buffer for this purpose, but passing + * this flag tells zstd to use the user provided buffer. + */ +#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 + +/* ZSTD_d_forceIgnoreChecksum + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * Tells the decompressor to skip checksum validation during decompression, regardless + * of whether checksumming was specified during compression. This offers some + * slight performance benefits, and may be useful for debugging. + * Param has values of type ZSTD_forceIgnoreChecksum_e + */ +#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 + +/* ZSTD_d_refMultipleDDicts + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * If enabled and dctx is allocated on the heap, then additional memory will be allocated + * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() + * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead + * store all references. At decompression time, the appropriate dictID is selected + * from the set of DDicts based on the dictID in the frame. + * + * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. + * + * Param has values of byte ZSTD_refMultipleDDicts_e + * + * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory + * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. + * Memory is allocated as per ZSTD_DCtx::customMem. + * + * Although this function allocates memory for the table, the user is still responsible for + * memory management of the underlying ZSTD_DDict* themselves. + */ +#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 + +/* ZSTD_d_disableHuffmanAssembly + * Set to 1 to disable the Huffman assembly implementation. + * The default value is 0, which allows zstd to use the Huffman assembly + * implementation if available. + * + * This parameter can be used to disable Huffman assembly at runtime. + * If you want to disable it at compile time you can define the macro + * ZSTD_DISABLE_ASM. + */ +#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5 + +/* ZSTD_d_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * Forces the decompressor to reject blocks whose content size is + * larger than the configured maxBlockSize. When maxBlockSize is + * larger than the windowSize, the windowSize is used instead. + * This saves memory on the decoder when you know all blocks are small. + * + * This option is typically used in conjunction with ZSTD_c_maxBlockSize. + * + * WARNING: This causes the decoder to reject otherwise valid frames + * that have block sizes larger than the configured maxBlockSize. + */ +#define ZSTD_d_maxBlockSize ZSTD_d_experimentalParam6 + + +/*! ZSTD_DCtx_setFormat() : + * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). + * Instruct the decoder context about what kind of data to decode next. + * This instruction is mandatory to decode data without a fully-formed header, + * such ZSTD_f_zstd1_magicless for example. + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ +ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +ZSTDLIB_STATIC_API +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); + +/*! ZSTD_decompressStream_simpleArgs() : + * Same as ZSTD_decompressStream(), + * but using only integral types as arguments. + * This can be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos); + + +/******************************************************************** +* Advanced streaming functions +* Warning : most of these functions are now redundant with the Advanced API. +* Once Advanced API reaches "stable" status, +* redundant functions will be deprecated, and then at some point removed. +********************************************************************/ + +/*===== Advanced Streaming compression functions =====*/ + +/*! ZSTD_initCStream_srcSize() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * pledgedSrcSize must be correct. If it is not known at init time, use + * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + * "0" also disables frame content size field. It may be enabled in the future. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, + int compressionLevel, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingDict() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * Creates of an internal CDict (incompatible with static CCtx), except if + * dict == NULL or dictSize < 8, in which case no dict is used. + * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if + * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + int compressionLevel); + +/*! ZSTD_initCStream_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParams(zcs, params); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. + * pledgedSrcSize must be correct. + * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingCDict() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * note : cdict will just be referenced, and must outlive compression session + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); + +/*! ZSTD_initCStream_usingCDict_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setFParams(zcs, fParams); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); + +/*! ZSTD_resetCStream() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + * explicitly specified. + * + * start a new frame, using same parameters from previous frame. + * This is typically useful to skip dictionary loading stage, since it will reuse it in-place. + * Note that zcs must be init at least once before using ZSTD_resetCStream(). + * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. + * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, + * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); + + +typedef struct { + unsigned long long ingested; /* nb input bytes read and buffered */ + unsigned long long consumed; /* nb input bytes actually compressed */ + unsigned long long produced; /* nb of compressed bytes generated and buffered */ + unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ + unsigned currentJobID; /* MT only : latest started job nb */ + unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ +} ZSTD_frameProgression; + +/* ZSTD_getFrameProgression() : + * tells how much data has been ingested (read from input) + * consumed (input actually compressed) and produced (output) for current frame. + * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. + * Aggregates progression inside active worker threads. + */ +ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); + +/*! ZSTD_toFlushNow() : + * Tell how many bytes are ready to be flushed immediately. + * Useful for multithreading scenarios (nbWorkers >= 1). + * Probe the oldest active job, defined as oldest job not yet entirely flushed, + * and check its output buffer. + * @return : amount of data stored in oldest job and ready to be flushed immediately. + * if @return == 0, it means either : + * + there is no active job (could be checked with ZSTD_frameProgression()), or + * + oldest job is still actively compressing data, + * but everything it has produced has also been flushed so far, + * therefore flush speed is limited by production speed of oldest job + * irrespective of the speed of concurrent (and newer) jobs. + */ +ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); + + +/*===== Advanced Streaming decompression functions =====*/ + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + * + * note: no dictionary will be used if dict == NULL or dictSize < 8 + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, ddict); + * + * note : ddict is referenced, it must outlive decompression session + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * + * reuse decompression parameters from previous init; saves dictionary loading + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); + + +/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API ********************* + * + * *** OVERVIEW *** + * The Block-Level Sequence Producer API allows users to provide their own custom + * sequence producer which libzstd invokes to process each block. The produced list + * of sequences (literals and matches) is then post-processed by libzstd to produce + * valid compressed blocks. + * + * This block-level offload API is a more granular complement of the existing + * frame-level offload API compressSequences() (introduced in v1.5.1). It offers + * an easier migration story for applications already integrated with libzstd: the + * user application continues to invoke the same compression functions + * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits + * from the specific advantages of the external sequence producer. For example, + * the sequence producer could be tuned to take advantage of known characteristics + * of the input, to offer better speed / ratio, or could leverage hardware + * acceleration not available within libzstd itself. + * + * See contrib/externalSequenceProducer for an example program employing the + * Block-Level Sequence Producer API. + * + * *** USAGE *** + * The user is responsible for implementing a function of type + * ZSTD_sequenceProducer_F. For each block, zstd will pass the following + * arguments to the user-provided function: + * + * - sequenceProducerState: a pointer to a user-managed state for the sequence + * producer. + * + * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer. + * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory + * backing outSeqs is managed by the CCtx. + * + * - src, srcSize: an input buffer for the sequence producer to parse. + * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX. + * + * - dict, dictSize: a history buffer, which may be empty, which the sequence + * producer may reference as it parses the src buffer. Currently, zstd will + * always pass dictSize == 0 into external sequence producers, but this will + * change in the future. + * + * - compressionLevel: a signed integer representing the zstd compression level + * set by the user for the current operation. The sequence producer may choose + * to use this information to change its compression strategy and speed/ratio + * tradeoff. Note: the compression level does not reflect zstd parameters set + * through the advanced API. + * + * - windowSize: a size_t representing the maximum allowed offset for external + * sequences. Note that sequence offsets are sometimes allowed to exceed the + * windowSize if a dictionary is present, see doc/zstd_compression_format.md + * for details. + * + * The user-provided function shall return a size_t representing the number of + * sequences written to outSeqs. This return value will be treated as an error + * code if it is greater than outSeqsCapacity. The return value must be non-zero + * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided + * for convenience, but any value greater than outSeqsCapacity will be treated as + * an error code. + * + * If the user-provided function does not return an error code, the sequences + * written to outSeqs must be a valid parse of the src buffer. Data corruption may + * occur if the parse is not valid. A parse is defined to be valid if the + * following conditions hold: + * - The sum of matchLengths and literalLengths must equal srcSize. + * - All sequences in the parse, except for the final sequence, must have + * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have + * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0. + * - All offsets must respect the windowSize parameter as specified in + * doc/zstd_compression_format.md. + * - If the final sequence has matchLength == 0, it must also have offset == 0. + * + * zstd will only validate these conditions (and fail compression if they do not + * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence + * validation has a performance cost. + * + * If the user-provided function returns an error, zstd will either fall back + * to an internal sequence producer or fail the compression operation. The user can + * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback + * cParam. Fallback compression will follow any other cParam settings, such as + * compression level, the same as in a normal compression operation. + * + * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F + * function by calling + * ZSTD_registerSequenceProducer(cctx, + * sequenceProducerState, + * sequenceProducer) + * This setting will persist until the next parameter reset of the CCtx. + * + * The sequenceProducerState must be initialized by the user before calling + * ZSTD_registerSequenceProducer(). The user is responsible for destroying the + * sequenceProducerState. + * + * *** LIMITATIONS *** + * This API is compatible with all zstd compression APIs which respect advanced parameters. + * However, there are three limitations: + * + * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported. + * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level + * external sequence producer. + * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some + * cases (see its documentation for details). Users must explicitly set + * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external + * sequence producer is registered. + * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default + * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should + * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence + * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog). + * + * Second, history buffers are not currently supported. Concretely, zstd will always pass + * dictSize == 0 to the external sequence producer (for now). This has two implications: + * - Dictionaries are not currently supported. Compression will *not* fail if the user + * references a dictionary, but the dictionary won't have any effect. + * - Stream history is not currently supported. All advanced compression APIs, including + * streaming APIs, work with external sequence producers, but each block is treated as + * an independent chunk without history from previous blocks. + * + * Third, multi-threading within a single compression is not currently supported. In other words, + * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered. + * Multi-threading across compressions is fine: simply create one CCtx per thread. + * + * Long-term, we plan to overcome all three limitations. There is no technical blocker to + * overcoming them. It is purely a question of engineering effort. + */ + +#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1)) + +typedef size_t (*ZSTD_sequenceProducer_F) ( + void* sequenceProducerState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +); + +/*! ZSTD_registerSequenceProducer() : + * Instruct zstd to use a block-level external sequence producer function. + * + * The sequenceProducerState must be initialized by the caller, and the caller is + * responsible for managing its lifetime. This parameter is sticky across + * compressions. It will remain set until the user explicitly resets compression + * parameters. + * + * Sequence producer registration is considered to be an "advanced parameter", + * part of the "advanced API". This means it will only have an effect on compression + * APIs which respect advanced parameters, such as compress2() and compressStream2(). + * Older compression APIs such as compressCCtx(), which predate the introduction of + * "advanced parameters", will ignore any external sequence producer setting. + * + * The sequence producer can be "cleared" by registering a NULL function pointer. This + * removes all limitations described above in the "LIMITATIONS" section of the API docs. + * + * The user is strongly encouraged to read the full API documentation (above) before + * calling this function. */ +ZSTDLIB_STATIC_API void +ZSTD_registerSequenceProducer( + ZSTD_CCtx* cctx, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); + +/*! ZSTD_CCtxParams_registerSequenceProducer() : + * Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params. + * This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(), + * which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx(). + * + * If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx() + * is required, then this function is for you. Otherwise, you probably don't need it. + * + * See tests/zstreamtest.c for example usage. */ +ZSTDLIB_STATIC_API void +ZSTD_CCtxParams_registerSequenceProducer( + ZSTD_CCtx_params* params, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); + + +/********************************************************************* +* Buffer-less and synchronous inner streaming functions (DEPRECATED) +* +* This API is deprecated, and will be removed in a future version. +* It allows streaming (de)compression with user allocated buffers. +* However, it is hard to use, and not as well tested as the rest of +* our API. +* +* Please use the normal streaming API instead: ZSTD_compressStream2, +* and ZSTD_decompressStream. +* If there is functionality that you need, but it doesn't provide, +* please open an issue on our GitHub. +********************************************************************* */ + +/** + Buffer-less streaming compression (synchronous mode) + + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. + ZSTD_CCtx object can be reused multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. + + Then, consume your input using ZSTD_compressContinue(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. + - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). + It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. + Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. + + `ZSTD_CCtx` object can be reused (ZSTD_compressBegin()) to compress again. +*/ + +/*===== Buffer-less streaming compression functions =====*/ +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ + +ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API +size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ +/** + Buffer-less streaming decompression (synchronous mode) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be reused multiple times. + + First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). + Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. + Data fragment must be large enough to ensure successful decoding. + `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. + result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least result bytes on next attempt. + errorCode, which can be tested using ZSTD_isError(). + + It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, + such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). + Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. + As a consequence, check that values remain within valid application range. + For example, do not allocate memory blindly, check that `windowSize` is within expectation. + Each application can set its own limits, depending on local restrictions. + For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. + + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference distance. + There are multiple ways to guarantee this condition. + + The most memory efficient way is to use a round buffer of sufficient size. + Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), + which can return an error code if required value is too large for current system (in 32-bits mode). + In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, + up to the moment there is not enough room left in the buffer to guarantee decoding another full block, + which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. + At which point, decoding can resume from the beginning of the buffer. + Note that already decoded data stored in the buffer should be flushed before being overwritten. + + There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. + + Finally, if you control the compression process, you can also ignore all buffer size rules, + as long as the encoder and decoder progress in "lock-step", + aka use exactly the same buffer sizes, break contiguity at the same place, etc. + + Once buffers are setup, start decompression, with ZSTD_decompressBegin(). + If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). + + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + + result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTD_isError(). + + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). + This information is not required to properly decode a frame. + + == Special case : skippable frames == + + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by decompressor. + The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. + For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. +*/ + +/*===== Buffer-less streaming decompression functions =====*/ + +ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* misc */ +ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + + + + +/* ========================================= */ +/** Block level API (DEPRECATED) */ +/* ========================================= */ + +/*! + + This API is deprecated in favor of the regular compression API. + You can get the frame header down to 2 bytes by setting: + - ZSTD_c_format = ZSTD_f_zstd1_magicless + - ZSTD_c_contentSizeFlag = 0 + - ZSTD_c_checksumFlag = 0 + - ZSTD_c_dictIDFlag = 0 + + This API is not as well tested as our normal API, so we recommend not using it. + We will be removing it in a future version. If the normal API doesn't provide + the functionality you need, please open a GitHub issue. + + Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). + But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : any ZSTD_compressBegin*() variant, including with dictionary + + decompression : any ZSTD_decompressBegin*() variant, including with dictionary + - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + + If input is larger than a block size, it's necessary to split input data into multiple blocks + + For inputs larger than a single block, consider using regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! + ===> In which case, nothing is produced into `dst` ! + + User __must__ test for such outcome and deal directly with uncompressed data + + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. + Doing so would mess up with statistics history, leading to potential data corruption. + + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! + + In case of multiple successive blocks, should some of them be uncompressed, + decoder must be informed of their existence in order to follow proper history. + Use ZSTD_insertBlock() for such a case. +*/ + +/*===== Raw zstd block functions =====*/ +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ -ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); #endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ +#if defined (__cplusplus) +} +#endif diff --git a/src/bled/zstd_bits.h b/src/bled/zstd_bits.h new file mode 100644 index 00000000000..8d89111a8de --- /dev/null +++ b/src/bled/zstd_bits.h @@ -0,0 +1,208 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_BITS_H +#define ZSTD_BITS_H + +#include "zstd_mem.h" + +MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3, + 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, + 26, 12, 18, 6, 11, 5, 10, 9}; + return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val) +{ + assert(val != 0); +# if defined(_MSC_VER) +# if STATIC_BMI2 == 1 + return (unsigned)_tzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_ctz(val); +# elif defined(__ICCARM__) + return (unsigned)__builtin_ctz(val); +# else + return ZSTD_countTrailingZeros32_fallback(val); +# endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) { + assert(val != 0); + { + static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31}; + val |= val >> 1; + val |= val >> 2; + val |= val >> 4; + val |= val >> 8; + val |= val >> 16; + return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val) +{ + assert(val != 0); +# if defined(_MSC_VER) +# if STATIC_BMI2 == 1 + return (unsigned)_lzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)(31 - r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_clz(val); +# elif defined(__ICCARM__) + return (unsigned)__builtin_clz(val); +# else + return ZSTD_countLeadingZeros32_fallback(val); +# endif +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val) +{ + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 == 1 + return (unsigned)_tzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__) + return (unsigned)__builtin_ctzll(val); +# elif defined(__ICCARM__) + return (unsigned)__builtin_ctzll(val); +# else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (leastSignificantWord == 0) { + return 32 + ZSTD_countTrailingZeros32(mostSignificantWord); + } else { + return ZSTD_countTrailingZeros32(leastSignificantWord); + } + } +# endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val) +{ + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 == 1 + return (unsigned)_lzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse64(&r, val); + return (unsigned)(63 - r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)(__builtin_clzll(val)); +# elif defined(__ICCARM__) + return (unsigned)(__builtin_clzll(val)); +# else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (mostSignificantWord == 0) { + return 32 + ZSTD_countLeadingZeros32(leastSignificantWord); + } else { + return ZSTD_countLeadingZeros32(mostSignificantWord); + } + } +# endif +} + +MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { + return ZSTD_countTrailingZeros64((U64)val) >> 3; + } else { + return ZSTD_countTrailingZeros32((U32)val) >> 3; + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { + return ZSTD_countLeadingZeros64((U64)val) >> 3; + } else { + return ZSTD_countLeadingZeros32((U32)val) >> 3; + } + } +} + +MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + return 31 - ZSTD_countLeadingZeros32(val); +} + +/* ZSTD_rotateRight_*(): + * Rotates a bitfield to the right by "count" bits. + * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts + */ +MEM_STATIC +U64 ZSTD_rotateRight_U64(U64 const value, U32 count) { + assert(count < 64); + count &= 0x3F; /* for fickle pattern recognition */ + return (value >> count) | (U64)(value << ((0U - count) & 0x3F)); +} + +MEM_STATIC +U32 ZSTD_rotateRight_U32(U32 const value, U32 count) { + assert(count < 32); + count &= 0x1F; /* for fickle pattern recognition */ + return (value >> count) | (U32)(value << ((0U - count) & 0x1F)); +} + +MEM_STATIC +U16 ZSTD_rotateRight_U16(U16 const value, U32 count) { + assert(count < 16); + count &= 0x0F; /* for fickle pattern recognition */ + return (value >> count) | (U16)(value << ((0U - count) & 0x0F)); +} + +#endif /* ZSTD_BITS_H */ diff --git a/src/bled/zstd_common.c b/src/bled/zstd_common.c new file mode 100644 index 00000000000..8c9909d01c1 --- /dev/null +++ b/src/bled/zstd_common.c @@ -0,0 +1,48 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + + +/*-************************************* +* Dependencies +***************************************/ +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_error_private.h" +#include "zstd_internal.h" + + +/*-**************************************** +* Version +******************************************/ +unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; } + +const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } + + +/*-**************************************** +* ZSTD Error Management +******************************************/ +#undef ZSTD_isError /* defined within zstd_internal.h */ +/*! ZSTD_isError() : + * tells if a return value is an error code + * symbol is required for external callers */ +unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } + +/*! ZSTD_getErrorName() : + * provides error code string from function result (useful for debugging) */ +const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } + +/*! ZSTD_getError() : + * convert a `size_t` function result into a proper ZSTD_errorCode enum */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } + +/*! ZSTD_getErrorString() : + * provides error code string from enum */ +const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } diff --git a/src/bled/zstd_compiler.h b/src/bled/zstd_compiler.h index 0064dac07f0..2ece5b89680 100644 --- a/src/bled/zstd_compiler.h +++ b/src/bled/zstd_compiler.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -11,6 +11,9 @@ #ifndef ZSTD_COMPILER_H #define ZSTD_COMPILER_H +#include <stddef.h> +#include "zstd_deps.h" + /*-******************************************************* * Compiler specifics *********************************************************/ @@ -23,7 +26,7 @@ # define INLINE_KEYWORD #endif -#if defined(__GNUC__) || defined(__ICCARM__) +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define FORCE_INLINE_ATTR __attribute__((always_inline)) #elif defined(_MSC_VER) # define FORCE_INLINE_ATTR __forceinline @@ -40,7 +43,7 @@ /** On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). - This explictly marks such functions as __cdecl so that the code will still compile + This explicitly marks such functions as __cdecl so that the code will still compile if a CC other than __cdecl has been made the default. */ #if defined(_MSC_VER) @@ -49,12 +52,19 @@ # define WIN_CDECL #endif +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + /** * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant * parameters. They must be inlined for the compiler to eliminate the constant * branches. */ -#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR UNUSED_ATTR /** * HINT_INLINE is used to help the compiler generate better code. It is *not* * used for "templates", so it can be tweaked based on the compilers @@ -69,21 +79,37 @@ #if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 # define HINT_INLINE static INLINE_KEYWORD #else -# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR +# define HINT_INLINE FORCE_INLINE_TEMPLATE #endif -/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +/* "soft" inline : + * The compiler is free to select if it's a good idea to inline or not. + * The main objective is to silence compiler warnings + * when a defined function in included but not used. + * + * Note : this macro is prefixed `MEM_` because it used to be provided by `mem.h` unit. + * Updating the prefix is probably preferable, but requires a fairly large codemod, + * since this name is used everywhere. + */ +#ifndef MEM_STATIC /* already defined in Linux Kernel mem.h */ #if defined(__GNUC__) -# define UNUSED_ATTR __attribute__((unused)) +# define MEM_STATIC static __inline UNUSED_ATTR +#elif defined(__IAR_SYSTEMS_ICC__) +# define MEM_STATIC static inline UNUSED_ATTR +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline #else -# define UNUSED_ATTR +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif #endif /* force no inlining */ #ifdef _MSC_VER # define FORCE_NOINLINE static __declspec(noinline) #else -# if defined(__GNUC__) || defined(__ICCARM__) +# if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define FORCE_NOINLINE static __attribute__((__noinline__)) # else # define FORCE_NOINLINE static @@ -92,37 +118,25 @@ /* target attribute */ -#ifndef __has_attribute - #define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */ -#endif -#if defined(__GNUC__) || defined(__ICCARM__) +#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__) # define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) #else # define TARGET_ATTRIBUTE(target) #endif -/* Enable runtime BMI2 dispatch based on the CPU. - * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. +/* Target attribute for BMI2 dynamic dispatch. + * Enable lzcnt, bmi, and bmi2. + * We test for bmi1 & bmi2. lzcnt is included in bmi1. */ -#ifndef DYNAMIC_BMI2 - #if ((defined(__clang__) && __has_attribute(__target__)) \ - || (defined(__GNUC__) \ - && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ - && (defined(__x86_64__) || defined(_M_X64)) \ - && !defined(__BMI2__) - # define DYNAMIC_BMI2 1 - #else - # define DYNAMIC_BMI2 0 - #endif -#endif +#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2") /* prefetch * can be disabled, by declaring NO_PREFETCH build macro */ #if defined(NO_PREFETCH) -# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ -# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */ +# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */ #else -# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) && !defined(_M_ARM64EC) /* _mm_prefetch() is not defined outside of x86/x64 */ # include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ # define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) # define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) @@ -130,24 +144,25 @@ # define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) # define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) # elif defined(__aarch64__) -# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) -# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) +# define PREFETCH_L1(ptr) do { __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))); } while (0) +# define PREFETCH_L2(ptr) do { __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))); } while (0) # else -# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ -# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */ +# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */ # endif #endif /* NO_PREFETCH */ #define CACHELINE_SIZE 64 -#define PREFETCH_AREA(p, s) { \ - const char* const _ptr = (const char*)(p); \ - size_t const _size = (size_t)(s); \ - size_t _pos; \ - for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ - PREFETCH_L2(_ptr + _pos); \ - } \ -} +#define PREFETCH_AREA(p, s) \ + do { \ + const char* const _ptr = (const char*)(p); \ + size_t const _size = (size_t)(s); \ + size_t _pos; \ + for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ + PREFETCH_L2(_ptr + _pos); \ + } \ + } while (0) /* vectorization * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax, @@ -175,6 +190,12 @@ #define UNLIKELY(x) (x) #endif +#if 0 || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))) +# define ZSTD_UNREACHABLE do { assert(0), __builtin_unreachable(); } while (0) +#else +# define ZSTD_UNREACHABLE do { assert(0); } while (0) +#endif + /* disable warnings */ #ifdef _MSC_VER /* Visual Studio */ # include <intrin.h> /* For Visual 2005 */ @@ -191,6 +212,8 @@ # ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 # define STATIC_BMI2 1 # endif +# elif defined(__BMI2__) && defined(__x86_64__) && defined(__GNUC__) +# define STATIC_BMI2 1 # endif #endif @@ -199,15 +222,19 @@ #endif /* compile time determination of SIMD support */ - -/* compat. with non-clang compilers */ -#ifndef __has_builtin -# define __has_builtin(x) 0 -#endif - -/* compat. with non-clang compilers */ -#ifndef __has_feature -# define __has_feature(x) 0 +#if !defined(ZSTD_NO_INTRINSICS) +# if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) +# define ZSTD_ARCH_X86_SSE2 +# endif +# if defined(__ARM_NEON) || defined(_M_ARM64) +# define ZSTD_ARCH_ARM_NEON +# endif +# +# if defined(ZSTD_ARCH_X86_SSE2) +# include <emmintrin.h> +# elif defined(ZSTD_ARCH_ARM_NEON) +# include <arm_neon.h> +# endif #endif /* C-language Attributes are added in C23. */ @@ -220,7 +247,11 @@ /* Only use C++ attributes in C++. Some compilers report support for C++ * attributes when compiling with C. */ -#define ZSTD_HAS_CPP_ATTRIBUTE(x) 0 +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0 +#endif /* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute. * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough @@ -233,10 +264,194 @@ # elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough) # define ZSTD_FALLTHROUGH [[fallthrough]] # elif __has_attribute(__fallthrough__) -# define ZSTD_FALLTHROUGH __attribute__((__fallthrough__)) +/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon + * gcc complains about: a label can only be part of a statement and a declaration is not a statement. + */ +# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__)) # else # define ZSTD_FALLTHROUGH # endif #endif +/*-************************************************************** +* Alignment check +*****************************************************************/ + +/* @return 1 if @u is a 2^n value, 0 otherwise + * useful to check a value is valid for alignment restrictions */ +MEM_STATIC int ZSTD_isPower2(size_t u) { + return (u & (u-1)) == 0; +} + +/* this test was initially positioned in mem.h, + * but this file is removed (or replaced) for linux kernel + * so it's now hosted in compiler.h, + * which remains valid for both user & kernel spaces. + */ + +#ifndef ZSTD_ALIGNOF +# if defined(__GNUC__) || defined(_MSC_VER) +/* covers gcc, clang & MSVC */ +/* note : this section must come first, before C11, + * due to a limitation in the kernel source generator */ +# define ZSTD_ALIGNOF(T) __alignof(T) + +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +/* C11 support */ +# include <stdalign.h> +# define ZSTD_ALIGNOF(T) alignof(T) + +# else +/* No known support for alignof() - imperfect backup */ +# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T)) + +# endif +#endif /* ZSTD_ALIGNOF */ + +/*-************************************************************** +* Sanitizer +*****************************************************************/ + +/** + * Zstd relies on pointer overflow in its decompressor. + * We add this attribute to functions that rely on pointer overflow. + */ +#ifndef ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +# if __has_attribute(no_sanitize) +# if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 8 + /* gcc < 8 only has signed-integer-overlow which triggers on pointer overflow */ +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("signed-integer-overflow"))) +# else + /* older versions of clang [3.7, 5.0) will warn that pointer-overflow is ignored. */ +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("pointer-overflow"))) +# endif +# else +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +# endif +#endif + +/** + * Helper function to perform a wrapped pointer difference without triggering + * UBSAN. + * + * @returns lhs - rhs with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +ptrdiff_t ZSTD_wrappedPtrDiff(unsigned char const* lhs, unsigned char const* rhs) +{ + return lhs - rhs; +} + +/** + * Helper function to perform a wrapped pointer add without triggering UBSAN. + * + * @return ptr + add with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +unsigned char const* ZSTD_wrappedPtrAdd(unsigned char const* ptr, ptrdiff_t add) +{ + return ptr + add; +} + +/** + * Helper function to perform a wrapped pointer subtraction without triggering + * UBSAN. + * + * @return ptr - sub with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +unsigned char const* ZSTD_wrappedPtrSub(unsigned char const* ptr, ptrdiff_t sub) +{ + return ptr - sub; +} + +/** + * Helper function to add to a pointer that works around C's undefined behavior + * of adding 0 to NULL. + * + * @returns `ptr + add` except it defines `NULL + 0 == NULL`. + */ +MEM_STATIC +unsigned char* ZSTD_maybeNullPtrAdd(unsigned char* ptr, ptrdiff_t add) +{ + return add > 0 ? ptr + add : ptr; +} + +/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an + * abundance of caution, disable our custom poisoning on mingw. */ +#ifdef __MINGW32__ +#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE +#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1 +#endif +#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE +#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1 +#endif +#endif + +#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE) +/* Not all platforms that support msan provide sanitizers/msan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include <stddef.h> /* size_t */ +#define ZSTD_DEPS_NEED_STDINT +#include "zstd_deps.h" /* intptr_t */ + +/* Make memory region fully initialized (without changing its contents). */ +void __msan_unpoison(const volatile void *a, size_t size); + +/* Make memory region fully uninitialized (without changing its contents). + This is a legacy interface that does not update origin information. Use + __msan_allocated_memory() instead. */ +void __msan_poison(const volatile void *a, size_t size); + +/* Returns the offset of the first (at least partially) poisoned byte in the + memory range, or -1 if the whole range is good. */ +intptr_t __msan_test_shadow(const volatile void *x, size_t size); + +/* Print shadow and origin for the memory range to stderr in a human-readable + format. */ +void __msan_print_shadow(const volatile void *x, size_t size); +#endif + +#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE) +/* Not all platforms that support asan provide sanitizers/asan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include <stddef.h> /* size_t */ + +/** + * Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable. + * + * This memory must be previously allocated by your program. Instrumented + * code is forbidden from accessing addresses in this region until it is + * unpoisoned. This function is not guaranteed to poison the entire region - + * it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan + * alignment restrictions. + * + * \note This function is not thread-safe because no two threads can poison or + * unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_poison_memory_region(void const volatile *addr, size_t size); + +/** + * Marks a memory region (<c>[addr, addr+size)</c>) as addressable. + * + * This memory must be previously allocated by your program. Accessing + * addresses in this region is allowed until this region is poisoned again. + * This function could unpoison a super-region of <c>[addr, addr+size)</c> due + * to ASan alignment restrictions. + * + * \note This function is not thread-safe because no two threads can + * poison or unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); +#endif + #endif /* ZSTD_COMPILER_H */ diff --git a/src/bled/zstd_config.h b/src/bled/zstd_config.h index 4792e1e2460..6ed38b23642 100644 --- a/src/bled/zstd_config.h +++ b/src/bled/zstd_config.h @@ -9,6 +9,7 @@ #define ZSTD_TRACE 0 #define ZSTD_DECOMPRESS_DICTIONARY 0 #define ZSTD_DECOMPRESS_MULTIFRAME 0 +#define ZSTD_NO_TRACE 1 #if CONFIG_FEATURE_ZSTD_SMALL >= 9 #define ZSTD_NO_INLINE 1 @@ -41,6 +42,16 @@ #pragma GCC diagnostic ignored "-Wunused-function" #endif +#ifndef __has_attribute +#define __has_attribute(x) 0 +#endif +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + /* Include zstd_deps.h first with all the options we need enabled. */ #define ZSTD_DEPS_NEED_MALLOC #define ZSTD_DEPS_NEED_MATH64 diff --git a/src/bled/zstd_cpu.h b/src/bled/zstd_cpu.h index 6866f60f464..13b4469dbfc 100644 --- a/src/bled/zstd_cpu.h +++ b/src/bled/zstd_cpu.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -35,6 +35,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { U32 f7b = 0; U32 f7c = 0; #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) +#if !defined(__clang__) || __clang_major__ >= 16 int reg[4]; __cpuid((int*)reg, 0); { @@ -50,6 +51,41 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { f7c = (U32)reg[2]; } } +#else + /* Clang compiler has a bug (fixed in https://reviews.llvm.org/D101338) in + * which the `__cpuid` intrinsic does not save and restore `rbx` as it needs + * to due to being a reserved register. So in that case, do the `cpuid` + * ourselves. Clang supports inline assembly anyway. + */ + U32 n; + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "popq %%rbx\n\t" + : "=a"(n) + : "a"(0) + : "rcx", "rdx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "popq %%rbx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1) + :); + } + if (n >= 7) { + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "movq %%rbx, %%rax\n\t" + "popq %%rbx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "rdx"); + } +#endif #elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) /* The following block like the normal cpuid branch below, but gcc * reserves ebx for use of its pic register so we must specially diff --git a/src/bled/zstd_ddict.c b/src/bled/zstd_ddict.c new file mode 100644 index 00000000000..6822b7bb75e --- /dev/null +++ b/src/bled/zstd_ddict.c @@ -0,0 +1,244 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_ddict.c : + * concentrates all logic that needs to know the internals of ZSTD_DDict object */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +//#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ +#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "zstd_cpu.h" /* bmi2 */ +#include "zstd_mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "huf.h" +#include "zstd_decompress_internal.h" +#include "zstd_ddict.h" + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "../legacy/zstd_legacy.h" +#endif + + + +/*-******************************************************* +* Types +*********************************************************/ +struct ZSTD_DDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictSize; + ZSTD_entropyDTables_t entropy; + U32 dictID; + U32 entropyPresent; + ZSTD_customMem cMem; +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictContent; +} + +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictSize; +} + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_copyDDictParameters"); + assert(dctx != NULL); + assert(ddict != NULL); + dctx->dictID = ddict->dictID; + dctx->prefixStart = ddict->dictContent; + dctx->virtualStart = ddict->dictContent; + dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; + dctx->previousDstEnd = dctx->dictEnd; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + if (ddict->entropyPresent) { + dctx->litEntropy = 1; + dctx->fseEntropy = 1; + dctx->LLTptr = ddict->entropy.LLTable; + dctx->MLTptr = ddict->entropy.MLTable; + dctx->OFTptr = ddict->entropy.OFTable; + dctx->HUFptr = ddict->entropy.hufTable; + dctx->entropy.rep[0] = ddict->entropy.rep[0]; + dctx->entropy.rep[1] = ddict->entropy.rep[1]; + dctx->entropy.rep[2] = ddict->entropy.rep[2]; + } else { + dctx->litEntropy = 0; + dctx->fseEntropy = 0; + } +} + + +static size_t +ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, + ZSTD_dictContentType_e dictContentType) +{ + ddict->dictID = 0; + ddict->entropyPresent = 0; + if (dictContentType == ZSTD_dct_rawContent) return 0; + + if (ddict->dictSize < 8) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + { U32 const magic = MEM_readLE32(ddict->dictContent); + if (magic != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + } + ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy( + &ddict->entropy, ddict->dictContent, ddict->dictSize)), + dictionary_corrupted, ""); + ddict->entropyPresent = 1; + return 0; +} + + +static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { + ddict->dictBuffer = NULL; + ddict->dictContent = dict; + if (!dict) dictSize = 0; + } else { + void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem); + ddict->dictBuffer = internalBuffer; + ddict->dictContent = internalBuffer; + if (!internalBuffer) return ERROR(memory_allocation); + ZSTD_memcpy(internalBuffer, dict, dictSize); + } + ddict->dictSize = dictSize; + ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ + + /* parse dictionary content */ + FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); + + return 0; +} + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem) +{ + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem); + if (ddict == NULL) return NULL; + ddict->cMem = customMem; + { size_t const initResult = ZSTD_initDDict_internal(ddict, + dict, dictSize, + dictLoadMethod, dictContentType); + if (ZSTD_isError(initResult)) { + ZSTD_freeDDict(ddict); + return NULL; + } } + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, to start decompression without startup delay. +* `dict` content is copied inside DDict. +* Consequently, `dict` can be released after `ZSTD_DDict` creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); +} + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, to start decompression without startup delay. + * Dictionary content is simply referenced, it will be accessed during decompression. + * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ +ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); +} + + +const ZSTD_DDict* ZSTD_initStaticDDict( + void* sBuffer, size_t sBufferSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + size_t const neededSpace = sizeof(ZSTD_DDict) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; + assert(sBuffer != NULL); + assert(dict != NULL); + if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ + if (sBufferSize < neededSpace) return NULL; + if (dictLoadMethod == ZSTD_dlm_byCopy) { + ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */ + dict = ddict+1; + } + if (ZSTD_isError( ZSTD_initDDict_internal(ddict, + dict, dictSize, + ZSTD_dlm_byRef, dictContentType) )) + return NULL; + return ddict; +} + + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = ddict->cMem; + ZSTD_customFree(ddict->dictBuffer, cMem); + ZSTD_customFree(ddict, cMem); + return 0; + } +} + +/*! ZSTD_estimateDDictSize() : + * Estimate amount of memory that will be needed to create a dictionary for decompression. + * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ +size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) +{ + return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); +} + +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; +} + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; + return ddict->dictID; +} diff --git a/src/bled/zstd_ddict.h b/src/bled/zstd_ddict.h index ca4dccbe5a2..e862b4c7a02 100644 --- a/src/bled/zstd_ddict.h +++ b/src/bled/zstd_ddict.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -34,16 +34,11 @@ * ZSTD_getDictID_fromDict() */ -MEM_STATIC const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); -MEM_STATIC size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); -MEM_STATIC void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); -#if ZSTD_DECOMPRESS_DICTIONARY == 0 -MEM_STATIC void ZSTD_clearDict(ZSTD_DCtx* dctx) { (void)dctx; } -MEM_STATIC size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) { (void)dctx; (void)ddict; return 0; } -MEM_STATIC size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output) { (void)zds; (void)output; return 0; } -MEM_STATIC ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) { (void)dctx; return NULL; } -#endif #endif /* ZSTD_DDICT_H */ diff --git a/src/bled/zstd_decompress.c b/src/bled/zstd_decompress.c index 98f2afb6470..fcf29f7545f 100644 --- a/src/bled/zstd_decompress.c +++ b/src/bled/zstd_decompress.c @@ -1,5 +1,5 @@ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -12,11 +12,23 @@ /* *************************************************************** * Tuning parameters *****************************************************************/ +/*! + * HEAPMODE : + * Select how default decompression function ZSTD_decompress() allocates its context, + * on stack (0), or into heap (1, default; requires malloc()). + * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected. + */ +#ifndef ZSTD_HEAPMODE +# define ZSTD_HEAPMODE 1 +#endif /*! * LEGACY_SUPPORT : * if set to 1+, ZSTD_decompress() can decode older formats (v0.1+) */ +#ifndef ZSTD_LEGACY_SUPPORT +# define ZSTD_LEGACY_SUPPORT 0 +#endif /*! * MAXWINDOWSIZE_DEFAULT : @@ -43,18 +55,162 @@ /*-******************************************************* * Dependencies *********************************************************/ -#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ -#include "zstd_cpu.h" /* bmi2 */ +#include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "zstd_error_private.h" +#include "zstd_internal.h" /* blockProperties_t */ #include "zstd_mem.h" /* low level memory routines */ +#include "zstd_bits.h" /* ZSTD_highbit32 */ #define FSE_STATIC_LINKING_ONLY #include "fse.h" -#define HUF_STATIC_LINKING_ONLY #include "huf.h" -#include "xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */ -#include "zstd_internal.h" /* blockProperties_t */ +#include "xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */ #include "zstd_decompress_internal.h" /* ZSTD_DCtx */ -#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ -#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "../legacy/zstd_legacy.h" +#endif + +/************************************* + * Multiple DDicts Hashset internals * + *************************************/ + +#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4 +#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. + * Currently, that means a 0.75 load factor. + * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded + * the load factor of the ddict hash set. + */ + +#define DDICT_HASHSET_TABLE_BASE_SIZE 64 +#define DDICT_HASHSET_RESIZE_FACTOR 2 + +/* Hash function to determine starting position of dict insertion within the table + * Returns an index between [0, hashSet->ddictPtrTableSize] + */ +static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) { + const U64 hash = XXH64(&dictID, sizeof(U32), 0); + /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */ + return hash & (hashSet->ddictPtrTableSize - 1); +} + +/* Adds DDict to a hashset without resizing it. + * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set. + * Returns 0 if successful, or a zstd error code if something went wrong. + */ +static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) { + const U32 dictID = ZSTD_getDictID_fromDDict(ddict); + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!"); + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + while (hashSet->ddictPtrTable[idx] != NULL) { + /* Replace existing ddict if inserting ddict with same dictID */ + if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) { + DEBUGLOG(4, "DictID already exists, replacing rather than adding"); + hashSet->ddictPtrTable[idx] = ddict; + return 0; + } + idx &= idxRangeMask; + idx++; + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + hashSet->ddictPtrTable[idx] = ddict; + hashSet->ddictPtrCount++; + return 0; +} + +/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and + * rehashes all values, allocates new table, frees old table. + * Returns 0 on success, otherwise a zstd error code. + */ +static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR; + const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem); + const ZSTD_DDict** oldTable = hashSet->ddictPtrTable; + size_t oldTableSize = hashSet->ddictPtrTableSize; + size_t i; + + DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize); + RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!"); + hashSet->ddictPtrTable = newTable; + hashSet->ddictPtrTableSize = newTableSize; + hashSet->ddictPtrCount = 0; + for (i = 0; i < oldTableSize; ++i) { + if (oldTable[i] != NULL) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), ""); + } + } + ZSTD_customFree((void*)oldTable, customMem); + DEBUGLOG(4, "Finished re-hash"); + return 0; +} + +/* Fetches a DDict with the given dictID + * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL. + */ +static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) { + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + for (;;) { + size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]); + if (currDictID == dictID || currDictID == 0) { + /* currDictID == 0 implies a NULL ddict entry */ + break; + } else { + idx &= idxRangeMask; /* Goes to start of table when we reach the end */ + idx++; + } + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + return hashSet->ddictPtrTable[idx]; +} + +/* Allocates space for and returns a ddict hash set + * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with. + * Returns NULL if allocation failed. + */ +static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) { + ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem); + DEBUGLOG(4, "Allocating new hash set"); + if (!ret) + return NULL; + ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem); + if (!ret->ddictPtrTable) { + ZSTD_customFree(ret, customMem); + return NULL; + } + ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; + ret->ddictPtrCount = 0; + return ret; +} + +/* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself. + * Note: The ZSTD_DDict* within the table are NOT freed. + */ +static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + DEBUGLOG(4, "Freeing ddict hash set"); + if (hashSet && hashSet->ddictPtrTable) { + ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem); + } + if (hashSet) { + ZSTD_customFree(hashSet, customMem); + } +} + +/* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set. + * Returns 0 on success, or a ZSTD error. + */ +static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) { + DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize); + if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), ""); + } + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), ""); + return 0; +} /*-************************************************************* * Context management @@ -63,12 +219,13 @@ size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; /* support sizeof NULL */ return sizeof(*dctx) -#if ZSTD_DECOMPRESS_DICTIONARY != 0 + ZSTD_sizeof_DDict(dctx->ddictLocal) -#endif + dctx->inBuffSize + dctx->outBuffSize; } +size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } + + static size_t ZSTD_startingInputLength(ZSTD_format_e format) { size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format); @@ -82,34 +239,53 @@ static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) assert(dctx->streamStage == zdss_init); dctx->format = ZSTD_f_zstd1; dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; - // dctx->outBufferMode = ZSTD_bm_buffered; REMOVED + dctx->outBufferMode = ZSTD_bm_buffered; dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; -#if ZSTD_DECOMPRESS_DICTIONARY != 0 dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict; -#endif + dctx->disableHufAsm = 0; + dctx->maxBlockSizeParam = 0; } static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) { - // dctx->staticSize = 0; REMOVED -#if ZSTD_DECOMPRESS_DICTIONARY != 0 + dctx->staticSize = 0; dctx->ddict = NULL; dctx->ddictLocal = NULL; dctx->dictEnd = NULL; dctx->ddictIsCold = 0; dctx->dictUses = ZSTD_dont_use; - dctx->ddictSet = NULL; -#endif dctx->inBuff = NULL; dctx->inBuffSize = 0; dctx->outBuffSize = 0; dctx->streamStage = zdss_init; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + dctx->legacyContext = NULL; + dctx->previousLegacyVersion = 0; +#endif dctx->noForwardProgress = 0; dctx->oversizedDuration = 0; + dctx->isFrameDecompression = 1; #if DYNAMIC_BMI2 - dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + dctx->bmi2 = ZSTD_cpuSupportsBmi2(); #endif + dctx->ddictSet = NULL; ZSTD_DCtx_resetParameters(dctx); +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentEndForFuzzing = NULL; +#endif +} + +ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) +{ + ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */ + + ZSTD_initDCtx_internal(dctx); + dctx->staticSize = workspaceSize; + dctx->inBuff = (char*)(dctx+1); + return dctx; } static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) { @@ -117,27 +293,47 @@ static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) { { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem); if (!dctx) return NULL; + dctx->customMem = customMem; ZSTD_initDCtx_internal(dctx); return dctx; } } +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + +ZSTD_DCtx* ZSTD_createDCtx(void) +{ + DEBUGLOG(3, "ZSTD_createDCtx"); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); +} + +static void ZSTD_clearDict(ZSTD_DCtx* dctx) +{ + ZSTD_freeDDict(dctx->ddictLocal); + dctx->ddictLocal = NULL; + dctx->ddict = NULL; + dctx->dictUses = ZSTD_dont_use; +} + size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; /* support free on NULL */ - RETURN_ERROR_IF(ZSTD_DCtx_get_staticSize(zds), memory_allocation, "not compatible with static DCtx"); - { ZSTD_customMem const cMem = ZSTD_defaultCMem; -#if ZSTD_DECOMPRESS_DICTIONARY != 0 + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx"); + { ZSTD_customMem const cMem = dctx->customMem; ZSTD_clearDict(dctx); -#endif ZSTD_customFree(dctx->inBuff, cMem); dctx->inBuff = NULL; -#if ZSTD_DECOMPRESS_DICTIONARY != 0 +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (dctx->legacyContext) + ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion); +#endif if (dctx->ddictSet) { ZSTD_freeDDictHashSet(dctx->ddictSet, cMem); dctx->ddictSet = NULL; } -#endif ZSTD_customFree(dctx, cMem); return 0; } @@ -150,10 +346,65 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ } +/* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on + * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then + * accordingly sets the ddict to be used to decompress the frame. + * + * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is. + * + * ZSTD_d_refMultipleDDicts must be enabled for this function to be called. + */ +static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) { + assert(dctx->refMultipleDDicts && dctx->ddictSet); + DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame"); + if (dctx->ddict) { + const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID); + if (frameDDict) { + DEBUGLOG(4, "DDict found!"); + ZSTD_clearDict(dctx); + dctx->dictID = dctx->fParams.dictID; + dctx->ddict = frameDDict; + dctx->dictUses = ZSTD_use_indefinitely; + } + } +} + + /*-************************************************************* * Frame header decoding ***************************************************************/ +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +unsigned ZSTD_isFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if (magic == ZSTD_MAGICNUMBER) return 1; + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(buffer, size)) return 1; +#endif + return 0; +} + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + */ +unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } + return 0; +} + /** ZSTD_frameHeaderSize_internal() : * srcSize must be large enough to reach header size fields. * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. @@ -174,21 +425,55 @@ static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZST } } +/** ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_frameHeaderSize_prefix. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +{ + return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1); +} + + /** ZSTD_getFrameHeader_advanced() : * decode Frame Header, or require larger `srcSize`. * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless * @return : 0, `zfhPtr` is correctly filled, * >0, `srcSize` is too small, value is wanted `srcSize` amount, - * or an error code, which can be tested using ZSTD_isError() */ +** or an error code, which can be tested using ZSTD_isError() */ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) { const BYTE* ip = (const BYTE*)src; size_t const minInputSize = ZSTD_startingInputLength(format); - ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ - if (srcSize < minInputSize) return minInputSize; - RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter"); + DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize); + + if (srcSize > 0) { + /* note : technically could be considered an assert(), since it's an invalid entry */ + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0"); + } + if (srcSize < minInputSize) { + if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) { + /* when receiving less than @minInputSize bytes, + * control these bytes at least correspond to a supported magic number + * in order to error out early if they don't. + **/ + size_t const toCopy = MIN(4, srcSize); + unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER); + assert(src != NULL); + ZSTD_memcpy(hbuf, src, toCopy); + if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) { + /* not a zstd frame : let's check if it's a skippable frame */ + MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START); + ZSTD_memcpy(hbuf, src, toCopy); + if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) { + RETURN_ERROR(prefix_unknown, + "first bytes don't correspond to any supported magic number"); + } } } + return minInputSize; + } + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */ if ( (format != ZSTD_f_zstd1_magicless) && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { @@ -278,6 +563,12 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) { +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize); + return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; + } +#endif { ZSTD_frameHeader zfh; if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR; @@ -288,6 +579,116 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) } } } +static size_t readSkippableFrameSize(void const* src, size_t srcSize) +{ + size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE; + U32 sizeU32; + + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); + RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, + frameParameter_unsupported, ""); + { size_t const skippableSize = skippableHeaderSize + sizeU32; + RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); + return skippableSize; + } +} + +/*! ZSTD_readSkippableFrame() : + * Retrieves content of a skippable frame, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if this is not a valid skippable frame. + * + * @return : number of bytes written or a ZSTD error. + */ +size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, /* optional, can be NULL */ + const void* src, size_t srcSize) +{ + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + { U32 const magicNumber = MEM_readLE32(src); + size_t skippableFrameSize = readSkippableFrameSize(src, srcSize); + size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE; + + /* check input validity */ + RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, ""); + RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, ""); + + /* deliver payload */ + if (skippableContentSize > 0 && dst != NULL) + ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize); + if (magicVariant != NULL) + *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START; + return skippableContentSize; + } +} + +/** ZSTD_findDecompressedSize() : + * `srcSize` must be the exact length of some number of ZSTD compressed and/or + * skippable frames + * note: compatible with legacy mode + * @return : decompressed size of the frames contained */ +unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long totalDstSize = 0; + + while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) { + U32 const magicNumber = MEM_readLE32(src); + + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + if (ZSTD_isError(skippableSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } + + { unsigned long long const fcs = ZSTD_getFrameContentSize(src, srcSize); + if (fcs >= ZSTD_CONTENTSIZE_ERROR) return fcs; + + if (totalDstSize + fcs < totalDstSize) + return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */ + totalDstSize += fcs; + } + /* skip to next frame */ + { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); + if (ZSTD_isError(frameSrcSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(frameSrcSize <= srcSize); + + src = (const BYTE *)src + frameSrcSize; + srcSize -= frameSrcSize; + } + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + if (srcSize) return ZSTD_CONTENTSIZE_ERROR; + + return totalDstSize; +} + +/** ZSTD_getDecompressedSize() : + * compatible with legacy mode + * @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - frame content is empty + - decompressed size field is not present in frame header + - frame header unknown / not supported + - frame header not complete (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN); + return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret; +} + /** ZSTD_decodeFrameHeader() : * `headerSize` must be the size provided by ZSTD_frameHeaderSize(). @@ -296,25 +697,21 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) { size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format); - U32 dictID = 0; if (ZSTD_isError(result)) return result; /* invalid header */ RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small"); -#if ZSTD_DECOMPRESS_DICTIONARY != 0 /* Reference DDict requested by frame if dctx references multiple ddicts */ if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) { ZSTD_DCtx_selectFrameDDict(dctx); } - dictID = dctx->dictID; -#endif +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* Skip the dictID check in fuzzing mode, because it makes the search * harder. */ - - RETURN_ERROR_IF(dctx->fParams.dictID && (dictID != dctx->fParams.dictID), + RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID), dictionary_wrong, ""); - +#endif dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0; if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0); dctx->processedCSize += headerSize; @@ -329,10 +726,168 @@ static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) return frameSizeInfo; } +static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize, ZSTD_format_e format) +{ + ZSTD_frameSizeInfo frameSizeInfo; + ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) + return ZSTD_findFrameSizeInfoLegacy(src, srcSize); +#endif + + if (format == ZSTD_f_zstd1 && (srcSize >= ZSTD_SKIPPABLEHEADERSIZE) + && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); + assert(ZSTD_isError(frameSizeInfo.compressedSize) || + frameSizeInfo.compressedSize <= srcSize); + return frameSizeInfo; + } else { + const BYTE* ip = (const BYTE*)src; + const BYTE* const ipstart = ip; + size_t remainingSize = srcSize; + size_t nbBlocks = 0; + ZSTD_frameHeader zfh; + + /* Extract Frame Header */ + { size_t const ret = ZSTD_getFrameHeader_advanced(&zfh, src, srcSize, format); + if (ZSTD_isError(ret)) + return ZSTD_errorFrameSizeInfo(ret); + if (ret > 0) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + } + + ip += zfh.headerSize; + remainingSize -= zfh.headerSize; + + /* Iterate over each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) + return ZSTD_errorFrameSizeInfo(cBlockSize); + + if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + + ip += ZSTD_blockHeaderSize + cBlockSize; + remainingSize -= ZSTD_blockHeaderSize + cBlockSize; + nbBlocks++; + + if (blockProperties.lastBlock) break; + } + + /* Final frame content checksum */ + if (zfh.checksumFlag) { + if (remainingSize < 4) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + ip += 4; + } + + frameSizeInfo.nbBlocks = nbBlocks; + frameSizeInfo.compressedSize = (size_t)(ip - ipstart); + frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) + ? zfh.frameContentSize + : (unsigned long long)nbBlocks * zfh.blockSizeMax; + return frameSizeInfo; + } +} + +static size_t ZSTD_findFrameCompressedSize_advanced(const void *src, size_t srcSize, ZSTD_format_e format) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, format); + return frameSizeInfo.compressedSize; +} + +/** ZSTD_findFrameCompressedSize() : + * See docs in zstd.h + * Note: compatible with legacy mode */ +size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) +{ + return ZSTD_findFrameCompressedSize_advanced(src, srcSize, ZSTD_f_zstd1); +} + +/** ZSTD_decompressBound() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame or a skippable frame + * `srcSize` must be at least as large as the frame contained + * @return : the maximum decompressed size of the compressed source + */ +unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) +{ + unsigned long long bound = 0; + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ZSTD_CONTENTSIZE_ERROR; + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + bound += decompressedBound; + } + return bound; +} + +size_t ZSTD_decompressionMargin(void const* src, size_t srcSize) +{ + size_t margin = 0; + unsigned maxBlockSize = 0; + + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + ZSTD_frameHeader zfh; + + FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), ""); + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ERROR(corruption_detected); + + if (zfh.frameType == ZSTD_frame) { + /* Add the frame header to our margin */ + margin += zfh.headerSize; + /* Add the checksum to our margin */ + margin += zfh.checksumFlag ? 4 : 0; + /* Add 3 bytes per block */ + margin += 3 * frameSizeInfo.nbBlocks; + + /* Compute the max block size */ + maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax); + } else { + assert(zfh.frameType == ZSTD_skippableFrame); + /* Add the entire skippable frame size to our margin. */ + margin += compressedSize; + } + + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + } + + /* Add the max block size back to the margin. */ + margin += maxBlockSize; + + return margin; +} + /*-************************************************************* * Frame decoding ***************************************************************/ +/** ZSTD_insertBlock() : + * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize); + ZSTD_checkContinuity(dctx, blockStart, blockSize); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { @@ -342,7 +897,7 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, if (srcSize == 0) return 0; RETURN_ERROR(dstBuffer_null, ""); } - ZSTD_memcpy(dst, src, srcSize); + ZSTD_memmove(dst, src, srcSize); return srcSize; } @@ -361,10 +916,28 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming) { +#if ZSTD_TRACE + if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) { + ZSTD_Trace trace; + ZSTD_memset(&trace, 0, sizeof(trace)); + trace.version = ZSTD_VERSION_NUMBER; + trace.streaming = streaming; + if (dctx->ddict) { + trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict); + trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict); + trace.dictionaryIsCold = dctx->ddictIsCold; + } + trace.uncompressedSize = (size_t)uncompressedSize; + trace.compressedSize = (size_t)compressedSize; + trace.dctx = dctx; + ZSTD_trace_decompress_end(dctx->traceCtx, &trace); + } +#else (void)dctx; (void)uncompressedSize; (void)compressedSize; (void)streaming; +#endif } @@ -400,8 +973,13 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; } + /* Shrink the blockSizeMax if enabled */ + if (dctx->maxBlockSizeParam != 0) + dctx->fParams.blockSizeMax = MIN(dctx->fParams.blockSizeMax, (unsigned)dctx->maxBlockSizeParam); + /* Loop on each block */ while (1) { + BYTE* oBlockEnd = oend; size_t decodedSize; blockProperties_t blockProperties; size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); @@ -411,27 +989,48 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, remainingSrcSize -= ZSTD_blockHeaderSize; RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); + if (ip >= op && ip < oBlockEnd) { + /* We are decompressing in-place. Limit the output pointer so that we + * don't overwrite the block that we are currently reading. This will + * fail decompression if the input & output pointers aren't spaced + * far enough apart. + * + * This is important to set, even when the pointers are far enough + * apart, because ZSTD_decompressBlock_internal() can decide to store + * literals in the output buffer, after the block it is decompressing. + * Since we don't want anything to overwrite our input, we have to tell + * ZSTD_decompressBlock_internal to never write past ip. + * + * See ZSTD_allocateLiteralsBuffer() for reference. + */ + oBlockEnd = op + (ip - op); + } + switch(blockProperties.blockType) { case bt_compressed: - decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1); + assert(dctx->isFrameDecompression == 1); + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, not_streaming); break; case bt_raw : + /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */ decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); break; case bt_rle : - decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize); + decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize); break; case bt_reserved : default: RETURN_ERROR(corruption_detected, "invalid block type"); } - - if (ZSTD_isError(decodedSize)) return decodedSize; - if (dctx->validateChecksum) + FORWARD_IF_ERROR(decodedSize, "Block decompression failure"); + DEBUGLOG(5, "Decompressed block of dSize = %u", (unsigned)decodedSize); + if (dctx->validateChecksum) { XXH64_update(&dctx->xxhState, op, decodedSize); - if (decodedSize != 0) + } + if (decodedSize) /* support dst = NULL,0 */ { op += decodedSize; + } assert(ip != NULL); ip += cBlockSize; remainingSrcSize -= cBlockSize; @@ -455,11 +1054,164 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, } ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); /* Allow caller to get size read */ + DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %i, consuming %i bytes of input", (int)(op-ostart), (int)(ip - (const BYTE*)*srcPtr)); *srcPtr = ip; *srcSizePtr = remainingSrcSize; return (size_t)(op-ostart); } +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + const ZSTD_DDict* ddict) +{ + void* const dststart = dst; + int moreThan1Frame = 0; + + DEBUGLOG(5, "ZSTD_decompressMultiFrame"); + assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */ + + if (ddict) { + dict = ZSTD_DDict_dictContent(ddict); + dictSize = ZSTD_DDict_dictSize(ddict); + } + + while (srcSize >= ZSTD_startingInputLength(dctx->format)) { + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (dctx->format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) { + size_t decodedSize; + size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); + if (ZSTD_isError(frameSize)) return frameSize; + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, + "legacy support is not compatible with static dctx"); + + decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + + { + unsigned long long const expectedSize = ZSTD_getFrameContentSize(src, srcSize); + RETURN_ERROR_IF(expectedSize == ZSTD_CONTENTSIZE_ERROR, corruption_detected, "Corrupted frame header!"); + if (expectedSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF(expectedSize != decodedSize, corruption_detected, + "Frame header size does not match decoded size!"); + } + } + + assert(decodedSize <= dstCapacity); + dst = (BYTE*)dst + decodedSize; + dstCapacity -= decodedSize; + + src = (const BYTE*)src + frameSize; + srcSize -= frameSize; + + continue; + } +#endif + + if (dctx->format == ZSTD_f_zstd1 && srcSize >= 4) { + U32 const magicNumber = MEM_readLE32(src); + DEBUGLOG(5, "reading magic number %08X", (unsigned)magicNumber); + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame detected : skip it */ + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + FORWARD_IF_ERROR(skippableSize, "invalid skippable frame"); + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; /* check next frame */ + } } + + if (ddict) { + /* we were called from ZSTD_decompress_usingDDict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), ""); + } else { + /* this will initialize correctly with no dict if dict == NULL, so + * use this in all cases but ddict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), ""); + } + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, + &src, &srcSize); + RETURN_ERROR_IF( + (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown) + && (moreThan1Frame==1), + srcSize_wrong, + "At least one frame successfully completed, " + "but following bytes are garbage: " + "it's more likely to be a srcSize error, " + "specifying more input bytes than size of frame(s). " + "Note: one could be unlucky, it might be a corruption error instead, " + "happening right at the place where we expect zstd magic bytes. " + "But this is _much_ less likely than a srcSize field error."); + if (ZSTD_isError(res)) return res; + assert(res <= dstCapacity); + if (res != 0) + dst = (BYTE*)dst + res; + dstCapacity -= res; + } + moreThan1Frame = 1; + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed"); + + return (size_t)((BYTE*)dst - (BYTE*)dststart); +} + +size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize) +{ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); +} + + +static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) +{ + switch (dctx->dictUses) { + default: + assert(0 /* Impossible */); + ZSTD_FALLTHROUGH; + case ZSTD_dont_use: + ZSTD_clearDict(dctx); + return NULL; + case ZSTD_use_indefinitely: + return dctx->ddict; + case ZSTD_use_once: + dctx->dictUses = ZSTD_dont_use; + return dctx->ddict; + } +} + +size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx)); +} + + +size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ +#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) + size_t regenSize; + ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem); + RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!"); + regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); + ZSTD_freeDCtx(dctx); + return regenSize; +#else /* stack mode */ + ZSTD_DCtx dctx; + ZSTD_initDCtx_internal(&dctx); + return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); +#endif +} + + /*-************************************** * Advanced Streaming Decompression API * Bufferless and synchronous @@ -467,8 +1219,8 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } /** - * Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed, - * we allow taking a partial block as the input. Currently only raw uncompressed blocks can + * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we + * allow taking a partial block as the input. Currently only raw uncompressed blocks can * be streamed. * * For blocks that can be streamed, this allows us to reduce the latency until we produce @@ -481,7 +1233,7 @@ static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t return dctx->expected; if (dctx->bType != bt_raw) return dctx->expected; - return MIN(MAX(inputSize, 1), dctx->expected); + return BOUNDED(1, inputSize, dctx->expected); } ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { @@ -587,7 +1339,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { case bt_compressed: DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); - rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1); + assert(dctx->isFrameDecompression == 1); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, is_streaming); dctx->expected = 0; /* Streaming not supported */ break; case bt_raw : @@ -656,6 +1409,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c case ZSTDds_decodeSkippableHeader: assert(src != NULL); assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + assert(dctx->format != ZSTD_f_zstd1_magicless); ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ dctx->stage = ZSTDds_skipFrame; @@ -668,7 +1422,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c default: assert(0); /* impossible */ - RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ } } @@ -679,6 +1433,10 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); dctx->prefixStart = dict; dctx->previousDstEnd = (const char*)dict + dictSize; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif return 0; } @@ -705,11 +1463,11 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, /* in minimal huffman, we always use X1 variants */ size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, - workspace, workspaceSize); + workspace, workspaceSize, /* flags */ 0); #else size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, dictPtr, (size_t)(dictEnd - dictPtr), - workspace, workspaceSize); + workspace, workspaceSize, /* flags */ 0); #endif RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); dictPtr += hSize; @@ -773,9 +1531,33 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, return (size_t)(dictPtr - (const BYTE*)dict); } +static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); + { U32 const magic = MEM_readLE32(dict); + if (magic != ZSTD_MAGIC_DICTIONARY) { + return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ + } } + dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); + RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, ""); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + dctx->litEntropy = dctx->fseEntropy = 1; + + /* reference dictionary content */ + return ZSTD_refDictContent(dctx, dict, dictSize); +} + size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) { assert(dctx != NULL); +#if ZSTD_TRACE + dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0; +#endif dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ dctx->stage = ZSTDds_getFrameHeaderSize; dctx->processedCSize = 0; @@ -784,12 +1566,11 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->prefixStart = NULL; dctx->virtualStart = NULL; dctx->dictEnd = NULL; - dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ dctx->litEntropy = dctx->fseEntropy = 0; -#if ZSTD_DECOMPRESS_DICTIONARY != 0 dctx->dictID = 0; -#endif dctx->bType = bt_reserved; + dctx->isFrameDecompression = 1; ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ dctx->LLTptr = dctx->entropy.LLTable; @@ -802,13 +1583,10 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); -#if ZSTD_DECOMPRESS_DICTIONARY != 0 if (dict && dictSize) RETURN_ERROR_IF( ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)), dictionary_corrupted, ""); -#endif - (void)dict; (void)dictSize; return 0; } @@ -819,8 +1597,6 @@ size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) { DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict"); assert(dctx != NULL); - (void)ddict; -#if ZSTD_DECOMPRESS_DICTIONARY != 0 if (ddict) { const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict); size_t const dictSize = ZSTD_DDict_dictSize(ddict); @@ -829,13 +1605,10 @@ size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) DEBUGLOG(4, "DDict is %s", dctx->ddictIsCold ? "~cold~" : "hot!"); } -#endif FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); -#if ZSTD_DECOMPRESS_DICTIONARY != 0 if (ddict) { /* NULL ddict is equivalent to no dictionary */ ZSTD_copyDDictParameters(dctx, ddict); } -#endif return 0; } @@ -856,7 +1629,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) * This could for one of the following reasons : * - The frame does not require a dictionary (most common case). * - The frame was built with dictID intentionally removed. - * Needed dictionary is a hidden information. + * Needed dictionary is a hidden piece of information. * Note : this use case also happens when using a non-conformant dictionary. * - `srcSize` is too small, and as a result, frame header could not be decoded. * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. @@ -865,12 +1638,28 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) * ZSTD_getFrameHeader(), which will provide a more precise error code. */ unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) { - ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 }; + ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 }; size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); if (ZSTD_isError(hError)) return 0; return zfp.dictID; } + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Use dictionary without significant overhead. */ +size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) +{ + /* pass content and size in case legacy frames are encountered */ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, + NULL, 0, + ddict); +} + + /*===================================== * Streaming decompression *====================================*/ @@ -881,6 +1670,16 @@ ZSTD_DStream* ZSTD_createDStream(void) return ZSTD_createDCtx_internal(ZSTD_defaultCMem); } +ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticDCtx(workspace, workspaceSize); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + size_t ZSTD_freeDStream(ZSTD_DStream* zds) { return ZSTD_freeDCtx(zds); @@ -892,12 +1691,63 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds) size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } +size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (dict && dictSize != 0) { + dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); + RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!"); + dctx->ddict = dctx->ddictLocal; + dctx->dictUses = ZSTD_use_indefinitely; + } + return 0; +} + +size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), ""); + dctx->dictUses = ZSTD_use_once; + return 0; +} + +size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + + +/* ZSTD_initDStream_usingDict() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + DEBUGLOG(4, "ZSTD_initDStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , ""); + return ZSTD_startingInputLength(zds->format); +} /* note : this variant can't fail */ size_t ZSTD_initDStream(ZSTD_DStream* zds) { DEBUGLOG(4, "ZSTD_initDStream"); - return ZSTD_initDStream_usingDDict(zds, NULL); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), ""); + FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), ""); + return ZSTD_startingInputLength(zds->format); } /* ZSTD_initDStream_usingDDict() : @@ -905,6 +1755,7 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) * this function cannot fail */ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) { + DEBUGLOG(4, "ZSTD_initDStream_usingDDict"); FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); return ZSTD_startingInputLength(dctx->format); @@ -915,11 +1766,178 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) * this function cannot fail */ size_t ZSTD_resetDStream(ZSTD_DStream* dctx) { + DEBUGLOG(4, "ZSTD_resetDStream"); FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); return ZSTD_startingInputLength(dctx->format); } +size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (ddict) { + dctx->ddict = ddict; + dctx->dictUses = ZSTD_use_indefinitely; + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) { + if (dctx->ddictSet == NULL) { + dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem); + if (!dctx->ddictSet) { + RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!"); + } + } + assert(!dctx->staticSize); /* Impossible: ddictSet cannot have been allocated if static dctx */ + FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), ""); + } + } + return 0; +} + +/* ZSTD_DCtx_setMaxWindowSize() : + * note : no direct equivalence in ZSTD_DCtx_setParameter, + * since this version sets windowSize, and the other sets windowLog */ +size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); + size_t const min = (size_t)1 << bounds.lowerBound; + size_t const max = (size_t)1 << bounds.upperBound; + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, ""); + RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, ""); + dctx->maxWindowSize = maxWindowSize; + return 0; +} + +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) +{ + return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format); +} + +ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + switch(dParam) { + case ZSTD_d_windowLogMax: + bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + case ZSTD_d_format: + bounds.lowerBound = (int)ZSTD_f_zstd1; + bounds.upperBound = (int)ZSTD_f_zstd1_magicless; + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + return bounds; + case ZSTD_d_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + case ZSTD_d_forceIgnoreChecksum: + bounds.lowerBound = (int)ZSTD_d_validateChecksum; + bounds.upperBound = (int)ZSTD_d_ignoreChecksum; + return bounds; + case ZSTD_d_refMultipleDDicts: + bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict; + bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts; + return bounds; + case ZSTD_d_disableHuffmanAssembly: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + case ZSTD_d_maxBlockSize: + bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; + bounds.upperBound = ZSTD_BLOCKSIZE_MAX; + return bounds; + + default:; + } + bounds.error = ERROR(parameter_unsupported); + return bounds; +} + +/* ZSTD_dParam_withinBounds: + * @return 1 if value is within dParam bounds, + * 0 otherwise */ +static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +#define CHECK_DBOUNDS(p,v) { \ + RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \ +} + +size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value) +{ + switch (param) { + case ZSTD_d_windowLogMax: + *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize); + return 0; + case ZSTD_d_format: + *value = (int)dctx->format; + return 0; + case ZSTD_d_stableOutBuffer: + *value = (int)dctx->outBufferMode; + return 0; + case ZSTD_d_forceIgnoreChecksum: + *value = (int)dctx->forceIgnoreChecksum; + return 0; + case ZSTD_d_refMultipleDDicts: + *value = (int)dctx->refMultipleDDicts; + return 0; + case ZSTD_d_disableHuffmanAssembly: + *value = (int)dctx->disableHufAsm; + return 0; + case ZSTD_d_maxBlockSize: + *value = dctx->maxBlockSizeParam; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + switch(dParam) { + case ZSTD_d_windowLogMax: + if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT; + CHECK_DBOUNDS(ZSTD_d_windowLogMax, value); + dctx->maxWindowSize = ((size_t)1) << value; + return 0; + case ZSTD_d_format: + CHECK_DBOUNDS(ZSTD_d_format, value); + dctx->format = (ZSTD_format_e)value; + return 0; + case ZSTD_d_stableOutBuffer: + CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value); + dctx->outBufferMode = (ZSTD_bufferMode_e)value; + return 0; + case ZSTD_d_forceIgnoreChecksum: + CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value); + dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value; + return 0; + case ZSTD_d_refMultipleDDicts: + CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value); + if (dctx->staticSize != 0) { + RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!"); + } + dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value; + return 0; + case ZSTD_d_disableHuffmanAssembly: + CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value); + dctx->disableHufAsm = value != 0; + return 0; + case ZSTD_d_maxBlockSize: + if (value != 0) CHECK_DBOUNDS(ZSTD_d_maxBlockSize, value); + dctx->maxBlockSizeParam = value; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) { @@ -927,6 +1945,7 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) || (reset == ZSTD_reset_session_and_parameters) ) { dctx->streamStage = zdss_init; dctx->noForwardProgress = 0; + dctx->isFrameDecompression = 1; } if ( (reset == ZSTD_reset_parameters) || (reset == ZSTD_reset_session_and_parameters) ) { @@ -937,10 +1956,23 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) return 0; } -size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) + +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) +{ + return ZSTD_sizeof_DCtx(dctx); +} + +static size_t ZSTD_decodingBufferSize_internal(unsigned long long windowSize, unsigned long long frameContentSize, size_t blockSizeMax) { - size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); - unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2); + size_t const blockSize = MIN((size_t)MIN(windowSize, ZSTD_BLOCKSIZE_MAX), blockSizeMax); + /* We need blockSize + WILDCOPY_OVERLENGTH worth of buffer so that if a block + * ends at windowSize + WILDCOPY_OVERLENGTH + 1 bytes, we can start writing + * the block at the beginning of the output buffer, and maintain a full window. + * + * We need another blockSize worth of buffer so that we can store split + * literals at the end of the block without overwriting the extDict window. + */ + unsigned long long const neededRBSize = windowSize + (blockSize * 2) + (WILDCOPY_OVERLENGTH * 2); unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); size_t const minRBSize = (size_t) neededSize; RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, @@ -948,6 +1980,31 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long return minRBSize; } +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + return ZSTD_decodingBufferSize_internal(windowSize, frameContentSize, ZSTD_BLOCKSIZE_MAX); +} + +size_t ZSTD_estimateDStreamSize(size_t windowSize) +{ + size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + size_t const inBuffSize = blockSize; /* no block can be larger */ + size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN); + return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize; +} + +size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) +{ + U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ + ZSTD_frameHeader zfh; + size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(err)) return err; + RETURN_ERROR_IF(err>0, srcSize_wrong, ""); + RETURN_ERROR_IF(zfh.windowSize > windowSizeMax, + frameParameter_windowTooLarge, ""); + return ZSTD_estimateDStreamSize((size_t)zfh.windowSize); +} + /* ***** Decompression ***** */ @@ -969,6 +2026,24 @@ static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds) return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; } +/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */ +static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output) +{ + ZSTD_outBuffer const expect = zds->expectedOutBuffer; + /* No requirement when ZSTD_obm_stable is not enabled. */ + if (zds->outBufferMode != ZSTD_bm_stable) + return 0; + /* Any buffer is allowed in zdss_init, this must be the same for every other call until + * the context is reset. + */ + if (zds->streamStage == zdss_init) + return 0; + /* The buffer must match our expectation exactly. */ + if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size) + return 0; + RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!"); +} + /* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream() * and updates the stage and the output buffer state. This call is extracted so it can be * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode. @@ -978,7 +2053,7 @@ static size_t ZSTD_decompressContinueStream( ZSTD_DStream* zds, char** op, char* oend, void const* src, size_t srcSize) { int const isSkipFrame = ZSTD_isSkipFrame(zds); - if (ZSTD_DCtx_get_outBufferMode(zds) == ZSTD_bm_buffered) { + if (zds->outBufferMode == ZSTD_bm_buffered) { size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart; size_t const decodedSize = ZSTD_decompressContinue(zds, zds->outBuff + zds->outStart, dstSize, src, srcSize); @@ -998,7 +2073,7 @@ static size_t ZSTD_decompressContinueStream( /* Flushing is not needed. */ zds->streamStage = zdss_read; assert(*op <= oend); - assert(ZSTD_DCtx_get_outBufferMode(zds) == ZSTD_bm_stable); + assert(zds->outBufferMode == ZSTD_bm_stable); } return 0; } @@ -1036,20 +2111,47 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB DEBUGLOG(5, "stage zdss_init => transparent reset "); zds->streamStage = zdss_loadHeader; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + zds->legacyVersion = 0; +#endif zds->hostageByte = 0; zds->expectedOutBuffer = *output; ZSTD_FALLTHROUGH; case zdss_loadHeader : DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + if (zds->legacyVersion) { + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); + if (hint==0) zds->streamStage = zdss_init; + return hint; + } } +#endif { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format); -#if ZSTD_DECOMPRESS_DICTIONARY != 0 if (zds->refMultipleDDicts && zds->ddictSet) { ZSTD_DCtx_selectFrameDDict(zds); } -#endif - DEBUGLOG(5, "header size : %u", (U32)hSize); if (ZSTD_isError(hSize)) { +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); + if (legacyVersion) { + ZSTD_DDict const* const ddict = ZSTD_getDDict(zds); + const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL; + size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0; + DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion); + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext, + zds->previousLegacyVersion, legacyVersion, + dict, dictSize), ""); + zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; + { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input); + if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */ + return hint; + } } +#endif return hSize; /* error */ } if (hSize != 0) { /* need more input */ @@ -1062,6 +2164,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->lhSize += remainingInput; } input->pos = input->size; + /* check first few bytes */ + FORWARD_IF_ERROR( + ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format), + "First few bytes detected incorrect" ); + /* return hint input size */ return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ } assert(ip != NULL); @@ -1069,8 +2176,27 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB break; } } + /* check for single-pass mode opportunity */ + if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && zds->fParams.frameType != ZSTD_skippableFrame + && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { + size_t const cSize = ZSTD_findFrameCompressedSize_advanced(istart, (size_t)(iend-istart), zds->format); + if (cSize <= (size_t)(iend-istart)) { + /* shortcut : using single-pass mode */ + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds)); + if (ZSTD_isError(decompressedSize)) return decompressedSize; + DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()"); + assert(istart != NULL); + ip = istart + cSize; + op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */ + zds->expected = 0; + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } } + /* Check output buffer is large enough for ZSTD_odm_stable. */ - if (ZSTD_DCtx_get_outBufferMode(zds) == ZSTD_bm_stable + if (zds->outBufferMode == ZSTD_bm_stable && zds->fParams.frameType != ZSTD_skippableFrame && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) { @@ -1081,7 +2207,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB DEBUGLOG(4, "Consume header"); FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); - if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + if (zds->format == ZSTD_f_zstd1 + && (MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); zds->stage = ZSTDds_skipFrame; } else { @@ -1097,11 +2224,13 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, frameParameter_windowTooLarge, ""); + if (zds->maxBlockSizeParam != 0) + zds->fParams.blockSizeMax = MIN(zds->fParams.blockSizeMax, (unsigned)zds->maxBlockSizeParam); /* Adapt buffer sizes to frame header instructions */ { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); - size_t const neededOutBuffSize = ZSTD_DCtx_get_outBufferMode(zds) == ZSTD_bm_buffered - ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize) + size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered + ? ZSTD_decodingBufferSize_internal(zds->fParams.windowSize, zds->fParams.frameContentSize, zds->fParams.blockSizeMax) : 0; ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); @@ -1115,17 +2244,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB (U32)zds->inBuffSize, (U32)neededInBuffSize); DEBUGLOG(4, "outBuff : from %u to %u", (U32)zds->outBuffSize, (U32)neededOutBuffSize); - if (ZSTD_DCtx_get_staticSize(zds)) { /* static DCtx */ - DEBUGLOG(4, "staticSize : %u", (U32)ZSTD_DCtx_get_staticSize(zds)); - assert(ZSTD_DCtx_get_staticSize(zds) >= sizeof(ZSTD_DCtx)); /* controlled at init */ + if (zds->staticSize) { /* static DCtx */ + DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); + assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ RETURN_ERROR_IF( - bufferSize > ZSTD_DCtx_get_staticSize(zds) - sizeof(ZSTD_DCtx), + bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), memory_allocation, ""); } else { - ZSTD_customFree(zds->inBuff, ZSTD_defaultCMem); + ZSTD_customFree(zds->inBuff, zds->customMem); zds->inBuffSize = 0; zds->outBuffSize = 0; - zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, ZSTD_defaultCMem); + zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem); RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, ""); } zds->inBuffSize = neededInBuffSize; @@ -1146,6 +2275,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); + assert(ip != NULL); ip += neededInSize; /* Function modifies the stage so we must break */ break; @@ -1160,7 +2290,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB int const isSkipFrame = ZSTD_isSkipFrame(zds); size_t loadedSize; /* At this point we shouldn't be decompressing a block that we can stream. */ - assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip)); + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip))); if (isSkipFrame) { loadedSize = MIN(toLoad, (size_t)(iend-ip)); } else { @@ -1169,8 +2299,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB "should never happen"); loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); } - ip += loadedSize; - zds->inPos += loadedSize; + if (loadedSize != 0) { + /* ip may be NULL */ + ip += loadedSize; + zds->inPos += loadedSize; + } if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ /* decode loaded input */ @@ -1180,14 +2313,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB break; } case zdss_flush: - { size_t const toFlushSize = zds->outEnd - zds->outStart; + { + size_t const toFlushSize = zds->outEnd - zds->outStart; size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); - op += flushedSize; + + op = op ? op + flushedSize : op; + zds->outStart += flushedSize; if (flushedSize == toFlushSize) { /* flush completed */ zds->streamStage = zdss_read; if ( (zds->outBuffSize < zds->fParams.frameContentSize) - && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", (int)(zds->outBuffSize - zds->outStart), (U32)zds->fParams.blockSizeMax); @@ -1201,7 +2337,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB default: assert(0); /* impossible */ - RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ } } /* result */ @@ -1214,8 +2350,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if ((ip==istart) && (op==ostart)) { /* no forward progress */ zds->noForwardProgress ++; if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { - RETURN_ERROR_IF(op==oend, dstSize_tooSmall, ""); - RETURN_ERROR_IF(ip==iend, srcSize_wrong, ""); + RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, ""); + RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, ""); assert(0); } } else { @@ -1246,3 +2382,23 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB return nextSrcSizeHint; } } + +size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos) +{ + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; + { size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } +} diff --git a/src/bled/zstd_decompress_block.c b/src/bled/zstd_decompress_block.c index 0f44c1ed415..1a35ed8eba1 100644 --- a/src/bled/zstd_decompress_block.c +++ b/src/bled/zstd_decompress_block.c @@ -1,5 +1,5 @@ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -26,6 +26,7 @@ #include "zstd_decompress_internal.h" /* ZSTD_DCtx */ #include "zstd_ddict.h" /* ZSTD_DDictDictContent */ #include "zstd_decompress_block.h" +#include "zstd_bits.h" /* ZSTD_highbit32 */ /*_******************************************************* * Macros @@ -51,6 +52,13 @@ static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } * Block decoding ***************************************************************/ +static size_t ZSTD_blockSizeMax(ZSTD_DCtx const* dctx) +{ + size_t const blockSizeMax = dctx->isFrameDecompression ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX; + assert(blockSizeMax <= ZSTD_BLOCKSIZE_MAX); + return blockSizeMax; +} + /*! ZSTD_getcBlockSize() : * Provides the size of compressed block from block header `src` */ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, @@ -69,21 +77,71 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, } } +/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */ +static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize, + const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately) +{ + size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); + assert(litSize <= blockSizeMax); + assert(dctx->isFrameDecompression || streaming == not_streaming); + assert(expectedWriteSize <= blockSizeMax); + if (streaming == not_streaming && dstCapacity > blockSizeMax + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) { + /* If we aren't streaming, we can just put the literals after the output + * of the current block. We don't need to worry about overwriting the + * extDict of our window, because it doesn't exist. + * So if we have space after the end of the block, just put it there. + */ + dctx->litBuffer = (BYTE*)dst + blockSizeMax + WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_in_dst; + } else if (litSize <= ZSTD_LITBUFFEREXTRASIZE) { + /* Literals fit entirely within the extra buffer, put them there to avoid + * having to split the literals. + */ + dctx->litBuffer = dctx->litExtraBuffer; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + } else { + assert(blockSizeMax > ZSTD_LITBUFFEREXTRASIZE); + /* Literals must be split between the output block and the extra lit + * buffer. We fill the extra lit buffer with the tail of the literals, + * and put the rest of the literals at the end of the block, with + * WILDCOPY_OVERLENGTH of buffer room to allow for overreads. + * This MUST not write more than our maxBlockSize beyond dst, because in + * streaming mode, that could overwrite part of our extDict window. + */ + if (splitImmediately) { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; + } else { + /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; + dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; + } + dctx->litBufferLocation = ZSTD_split; + assert(dctx->litBufferEnd <= (BYTE*)dst + expectedWriteSize); + } +} -/* Hidden declaration for fullbench */ -static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, - const void* src, size_t srcSize); /*! ZSTD_decodeLiteralsBlock() : + * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored + * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current + * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being + * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write. + * * @return : nb of bytes read from src (< srcSize ) * note : symbol not declared but exposed for fullbench */ -size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, - const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ +static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */ + void* dst, size_t dstCapacity, const streaming_operation streaming) { DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); { const BYTE* const istart = (const BYTE*) src; symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); switch(litEncType) { @@ -93,12 +151,16 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, ZSTD_FALLTHROUGH; case set_compressed: - RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3"); + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3"); { size_t lhSize, litSize, litCSize; U32 singleStream=0; U32 const lhlCode = (istart[0] >> 2) & 3; U32 const lhc = MEM_readLE32(istart); size_t hufSuccess; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); + int const flags = 0 + | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0) + | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0); switch(lhlCode) { case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -121,25 +183,31 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); break; } - RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + if (!singleStream) + RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong, + "Not enough literals (%zu) for the 4-streams mode (min %u)", + litSize, MIN_LITERALS_FOR_4_STREAMS); RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0); -#if ZSTD_DECOMPRESS_DICTIONARY != 0 /* prefetch huffman table if cold */ if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); } -#endif if (litEncType==set_repeat) { if (singleStream) { - hufSuccess = HUF_decompress1X_usingDTable_bmi2( + hufSuccess = HUF_decompress1X_usingDTable( dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx)); + dctx->HUFptr, flags); } else { - hufSuccess = HUF_decompress4X_usingDTable_bmi2( + assert(litSize >= MIN_LITERALS_FOR_4_STREAMS); + hufSuccess = HUF_decompress4X_usingDTable( dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx)); + dctx->HUFptr, flags); } } else { if (singleStream) { @@ -147,20 +215,29 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, hufSuccess = HUF_decompress1X_DCtx_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace)); + sizeof(dctx->workspace), flags); #else - hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2( + hufSuccess = HUF_decompress1X1_DCtx_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx)); + sizeof(dctx->workspace), flags); #endif } else { - hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2( + hufSuccess = HUF_decompress4X_hufOnly_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx)); + sizeof(dctx->workspace), flags); } } + if (dctx->litBufferLocation == ZSTD_split) + { + assert(litSize > ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE); + dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd -= WILDCOPY_OVERLENGTH; + assert(dctx->litBufferEnd <= (BYTE*)dst + blockSizeMax); + } RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); @@ -168,13 +245,13 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, dctx->litSize = litSize; dctx->litEntropy = 1; if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; - ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return litCSize + lhSize; } case set_basic: { size_t litSize, lhSize; U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); switch(lhlCode) { case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -187,27 +264,42 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, break; case 3: lhSize = 3; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3"); litSize = MEM_readLE24(istart) >> 4; break; } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); - ZSTD_memcpy(dctx->litBuffer, istart+lhSize, litSize); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize); + } dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; - ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); return lhSize+litSize; } /* direct reference into compressed stream */ dctx->litPtr = istart+lhSize; dctx->litSize = litSize; + dctx->litBufferEnd = dctx->litPtr + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; return lhSize+litSize; } case set_rle: { U32 const lhlCode = ((istart[0]) >> 2) & 3; size_t litSize, lhSize; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); switch(lhlCode) { case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -216,16 +308,28 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, break; case 1: lhSize = 2; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3"); litSize = MEM_readLE16(istart) >> 4; break; case 3: lhSize = 3; + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4"); litSize = MEM_readLE24(istart) >> 4; - RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4"); break; } - RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); - ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize); + } dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; return lhSize+1; @@ -236,6 +340,18 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, } } +/* Hidden declaration for fullbench */ +size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity); +size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity) +{ + dctx->isFrameDecompression = 0; + return ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, not_streaming); +} + /* Default FSE distribution tables. * These are pre-calculated FSE decoding tables using default distributions as defined in specification : * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions @@ -243,7 +359,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, * - start from default distributions, present in /lib/common/zstd_internal.h * - generate tables normally, using ZSTD_buildFSETable() * - printout the content of tables - * - pretify output, report below, test with fuzzer to ensure it's correct */ + * - prettify output, report below, test with fuzzer to ensure it's correct */ /* Default FSE distribution table for Literal Lengths */ static const ZSTD_seqSymbol LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = { @@ -345,7 +461,7 @@ static const ZSTD_seqSymbol ML_defaultDTable[(1<<ML_DEFAULTNORMLOG)+1] = { }; /* ML_defaultDTable */ -static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U32 nbAddBits) +static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U8 nbAddBits) { void* ptr = dt; ZSTD_seqSymbol_header* const DTableH = (ZSTD_seqSymbol_header*)ptr; @@ -357,7 +473,7 @@ static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U32 nbAddB cell->nbBits = 0; cell->nextState = 0; assert(nbAddBits < 255); - cell->nbAdditionalBits = (BYTE)nbAddBits; + cell->nbAdditionalBits = nbAddBits; cell->baseValue = baseValue; } @@ -369,7 +485,7 @@ static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U32 nbAddB FORCE_INLINE_TEMPLATE void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U32* nbAdditionalBits, + const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize) { ZSTD_seqSymbol* const tableDecode = dt+1; @@ -432,14 +548,15 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (i = 8; i < n; i += 8) { MEM_write64(spread + pos + i, sv); } - pos += n; + assert(n>=0); + pos += (size_t)n; } } /* Now we spread those positions across the table. - * The benefit of doing it in two stages is that we avoid the the + * The benefit of doing it in two stages is that we avoid the * variable size inner loop, which caused lots of branch misses. * Now we can run through all the positions without any branch misses. - * We unroll the loop twice, since that is what emperically worked best. + * We unroll the loop twice, since that is what empirically worked best. */ { size_t position = 0; @@ -466,7 +583,7 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (i=0; i<n; i++) { tableDecode[position].baseValue = s; position = (position + step) & tableMask; - while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ + while (UNLIKELY(position > highThreshold)) position = (position + step) & tableMask; /* lowprob area */ } } assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ } @@ -477,10 +594,10 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (u=0; u<tableSize; u++) { U32 const symbol = tableDecode[u].baseValue; U32 const nextState = symbolNext[symbol]++; - tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) ); + tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) ); tableDecode[u].nextState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); assert(nbAdditionalBits[symbol] < 255); - tableDecode[u].nbAdditionalBits = (BYTE)nbAdditionalBits[symbol]; + tableDecode[u].nbAdditionalBits = nbAdditionalBits[symbol]; tableDecode[u].baseValue = baseValue[symbol]; } } @@ -489,7 +606,7 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, /* Avoids the FORCE_INLINE of the _body() function. */ static void ZSTD_buildFSETable_body_default(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U32* nbAdditionalBits, + const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize) { ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue, @@ -497,9 +614,9 @@ static void ZSTD_buildFSETable_body_default(ZSTD_seqSymbol* dt, } #if DYNAMIC_BMI2 -TARGET_ATTRIBUTE("bmi2") static void ZSTD_buildFSETable_body_bmi2(ZSTD_seqSymbol* dt, +BMI2_TARGET_ATTRIBUTE static void ZSTD_buildFSETable_body_bmi2(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U32* nbAdditionalBits, + const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize) { ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue, @@ -509,7 +626,7 @@ TARGET_ATTRIBUTE("bmi2") static void ZSTD_buildFSETable_body_bmi2(ZSTD_seqSymbol void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U32* nbAdditionalBits, + const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize, int bmi2) { #if DYNAMIC_BMI2 @@ -531,7 +648,7 @@ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymbol** DTablePtr, symbolEncodingType_e type, unsigned max, U32 maxLog, const void* src, size_t srcSize, - const U32* baseValue, const U32* nbAdditionalBits, + const U32* baseValue, const U8* nbAdditionalBits, const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable, int ddictIsCold, int nbSeq, U32* wksp, size_t wkspSize, int bmi2) @@ -543,7 +660,7 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb RETURN_ERROR_IF((*(const BYTE*)src) > max, corruption_detected, ""); { U32 const symbol = *(const BYTE*)src; U32 const baseline = baseValue[symbol]; - U32 const nbBits = nbAdditionalBits[symbol]; + U8 const nbBits = nbAdditionalBits[symbol]; ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); } *DTablePtr = DTableSpace; @@ -590,11 +707,6 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, /* SeqHead */ nbSeq = *ip++; - if (!nbSeq) { - *nbSeqPtr=0; - RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, ""); - return 1; - } if (nbSeq > 0x7F) { if (nbSeq == 0xFF) { RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); @@ -607,16 +719,20 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, } *nbSeqPtr = nbSeq; + if (nbSeq == 0) { + /* No sequence : section ends immediately */ + RETURN_ERROR_IF(ip != iend, corruption_detected, + "extraneous data present in the Sequences section"); + return (size_t)(ip - istart); + } + /* FSE table descriptors */ RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ + RETURN_ERROR_IF(*ip & 3, corruption_detected, ""); /* The last field, Reserved, must be all-zeroes. */ { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); - int ddictIsCold = 0; ip++; -#if ZSTD_DECOMPRESS_DICTIONARY != 0 - ddictIsCold = dctx->ddictIsCold; -#endif /* Build DTables */ { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, @@ -624,7 +740,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, ip, iend-ip, LL_base, LL_bits, LL_defaultDTable, dctx->fseEntropy, - ddictIsCold, nbSeq, + dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); @@ -636,7 +752,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, ip, iend-ip, OF_base, OF_bits, OF_defaultDTable, dctx->fseEntropy, - ddictIsCold, nbSeq, + dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); @@ -648,7 +764,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, ip, iend-ip, ML_base, ML_bits, ML_defaultDTable, dctx->fseEntropy, - ddictIsCold, nbSeq, + dctx->ddictIsCold, nbSeq, dctx->workspace, sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx)); RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); @@ -719,7 +835,7 @@ HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. * The src buffer must be before the dst buffer. */ -static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { +static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { ptrdiff_t const diff = op - ip; BYTE* const oend = op + length; @@ -735,6 +851,7 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ assert(length >= 8); ZSTD_overlapCopy8(&op, &ip, diff); + length -= 8; assert(op - ip >= 8); assert(op <= oend); } @@ -749,12 +866,35 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ assert(oend > oend_w); ZSTD_wildcopy(op, ip, oend_w - op, ovtype); ip += oend_w - op; - op = oend_w; + op += oend_w - op; } /* Handle the leftovers. */ while (op < oend) *op++ = *ip++; } +/* ZSTD_safecopyDstBeforeSrc(): + * This version allows overlap with dst before src, or handles the non-overlap case with dst after src + * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */ +static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, ptrdiff_t length) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + if (length < 8 || diff > -8) { + /* Handle short lengths, close overlaps, and dst not before src. */ + while (op < oend) *op++ = *ip++; + return; + } + + if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) { + ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap); + ip += oend - WILDCOPY_OVERLENGTH - op; + op += oend - WILDCOPY_OVERLENGTH - op; + } + + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + /* ZSTD_execSequenceEnd(): * This version handles cases that are near the end of the output buffer. It requires * more careful checks to make sure there is no overflow. By separating out these hard @@ -764,10 +904,11 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_ * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). */ FORCE_NOINLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_execSequenceEnd(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; @@ -790,27 +931,78 @@ size_t ZSTD_execSequenceEnd(BYTE* op, if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix */ RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); - match = dictEnd - (prefixStart-match); + match = dictEnd - (prefixStart - match); if (match + sequence.matchLength <= dictEnd) { ZSTD_memmove(oLitEnd, match, sequence.matchLength); return sequenceLength; } /* span extDict & currentPrefixSegment */ { size_t const length1 = dictEnd - match; - ZSTD_memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = prefixStart; - } } + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +/* ZSTD_execSequenceEndSplitLitBuffer(): + * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case. + */ +FORCE_NOINLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer"); + ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); return sequenceLength; } HINT_INLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR size_t ZSTD_execSequence(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; @@ -821,6 +1013,104 @@ size_t ZSTD_execSequence(BYTE* op, assert(op != NULL /* Precondition */); assert(oend_w < oend /* No underflow */); + +#if defined(__aarch64__) + /* prefetch sequence starting from match that will be used for copy later */ + PREFETCH_L1(match); +#endif + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + +HINT_INLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); /* Handle edge cases in a slow path: * - Read beyond end of literals * - Match end is within WILDCOPY_OVERLIMIT of oend @@ -830,7 +1120,7 @@ size_t ZSTD_execSequence(BYTE* op, iLitEnd > litLimit || oMatchEnd > oend_w || (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) - return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ assert(op <= oLitEnd /* No overflow */); @@ -898,6 +1188,7 @@ size_t ZSTD_execSequence(BYTE* op, return sequenceLength; } + static void ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) { @@ -918,7 +1209,7 @@ ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 } /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum - * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1) + * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32 * bits before reloading. This value is the maximum number of bytes we read * after reloading when we are decoding long offsets. */ @@ -929,13 +1220,37 @@ ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; +/** + * ZSTD_decodeSequence(): + * @p longOffsets : tells the decoder to reload more bit while decoding large offsets + * only used in 32-bit mode + * @return : Sequence (litL + matchL + offset) + */ FORCE_INLINE_TEMPLATE seq_t -ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const int isLastSeq) { seq_t seq; + /* + * ZSTD_seqSymbol is a 64 bits wide structure. + * It can be loaded in one operation + * and its fields extracted by simply shifting or bit-extracting on aarch64. + * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh + * operations that cause performance drop. This can be avoided by using this + * ZSTD_memcpy hack. + */ +#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__)) + ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS; + ZSTD_seqSymbol* const llDInfo = &llDInfoS; + ZSTD_seqSymbol* const mlDInfo = &mlDInfoS; + ZSTD_seqSymbol* const ofDInfo = &ofDInfoS; + ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol)); +#else const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; +#endif seq.matchLength = mlDInfo->baseValue; seq.litLength = llDInfo->baseValue; { U32 const ofBase = ofDInfo->baseValue; @@ -950,28 +1265,31 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) U32 const llnbBits = llDInfo->nbBits; U32 const mlnbBits = mlDInfo->nbBits; U32 const ofnbBits = ofDInfo->nbBits; + + assert(llBits <= MaxLLBits); + assert(mlBits <= MaxMLBits); + assert(ofBits <= MaxOff); /* * As gcc has better branch and block analyzers, sometimes it is only - * valuable to mark likelyness for clang, it gives around 3-4% of + * valuable to mark likeliness for clang, it gives around 3-4% of * performance. */ /* sequence */ { size_t offset; - #if defined(__clang__) - if (LIKELY(ofBits > 1)) { - #else if (ofBits > 1) { - #endif ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); - assert(ofBits <= MaxOff); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits); if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { - U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed); + /* Always read extra bits, this keeps the logic simple, + * avoids branches, and avoids accidentally reading 0 bits. + */ + U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32; offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); BIT_reloadDStream(&seqState->DStream); - if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); - assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */ + offset += BIT_readBitsFast(&seqState->DStream, extraBits); } else { offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); @@ -988,7 +1306,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) } else { offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + temp -= !temp; /* 0 is not valid: input corrupted => force offset to -1 => corruption detected at execSequence */ if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset = temp; @@ -996,11 +1314,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) seq.offset = offset; } - #if defined(__clang__) - if (UNLIKELY(mlBits > 0)) - #else if (mlBits > 0) - #endif seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) @@ -1010,11 +1324,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - #if defined(__clang__) - if (UNLIKELY(llBits > 0)) - #else if (llBits > 0) - #endif seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); if (MEM_32bits()) @@ -1023,39 +1333,92 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); - ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + if (!isLastSeq) { + /* don't update FSE state for last Sequence */ + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + BIT_reloadDStream(&seqState->DStream); + } } return seq; } +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) +#if DEBUGLEVEL >= 1 +static int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) +{ + size_t const windowSize = dctx->fParams.windowSize; + /* No dictionary used. */ + if (dctx->dictContentEndForFuzzing == NULL) return 0; + /* Dictionary is our prefix. */ + if (prefixStart == dctx->dictContentBeginForFuzzing) return 1; + /* Dictionary is not our ext-dict. */ + if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0; + /* Dictionary is not within our window size. */ + if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0; + /* Dictionary is active. */ + return 1; +} +#endif + +static void ZSTD_assertValidSequence( + ZSTD_DCtx const* dctx, + BYTE const* op, BYTE const* oend, + seq_t const seq, + BYTE const* prefixStart, BYTE const* virtualStart) +{ +#if DEBUGLEVEL >= 1 + if (dctx->isFrameDecompression) { + size_t const windowSize = dctx->fParams.windowSize; + size_t const sequenceSize = seq.litLength + seq.matchLength; + BYTE const* const oLitEnd = op + seq.litLength; + DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + assert(op <= oend); + assert((size_t)(oend - op) >= sequenceSize); + assert(sequenceSize <= ZSTD_blockSizeMax(dctx)); + if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { + size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); + /* Offset must be within the dictionary. */ + assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); + assert(seq.offset <= windowSize + dictSize); + } else { + /* Offset must be within our window. */ + assert(seq.offset <= windowSize); + } + } +#else + (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart; +#endif +} +#endif #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + + FORCE_INLINE_TEMPLATE size_t DONT_VECTORIZE -ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, +ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { const BYTE* ip = (const BYTE*)seqStart; const BYTE* const iend = ip + seqSize; BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + maxDstSize; + BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, maxDstSize); BYTE* op = ostart; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* litBufferEnd = dctx->litBufferEnd; const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - DEBUGLOG(5, "ZSTD_decompressSequences_body"); - (void)frame; + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer (%i seqs)", nbSeq); - /* Regen sequences */ + /* Literals are split between internal buffer & output buffer */ if (nbSeq) { seqState_t seqState; dctx->fseEntropy = 1; @@ -1073,108 +1436,291 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, BIT_DStream_endOfBuffer < BIT_DStream_completed && BIT_DStream_completed < BIT_DStream_overflow); + /* decompress without overrunning litPtr begins */ + { seq_t sequence = {0,0,0}; /* some static analyzer believe that @sequence is not initialized (it necessarily is, since for(;;) loop as at least one iteration) */ + /* Align the decompression loop to 32 + 16 bytes. + * + * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression + * speed swings based on the alignment of the decompression loop. This + * performance swing is caused by parts of the decompression loop falling + * out of the DSB. The entire decompression loop should fit in the DSB, + * when it can't we get much worse performance. You can measure if you've + * hit the good case or the bad case with this perf command for some + * compressed file test.zst: + * + * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ + * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst + * + * If you see most cycles served out of the MITE you've hit the bad case. + * If you see most cycles served out of the DSB you've hit the good case. + * If it is pretty even then you may be in an okay case. + * + * This issue has been reproduced on the following CPUs: + * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 + * Use Instruments->Counters to get DSB/MITE cycles. + * I never got performance swings, but I was able to + * go from the good case of mostly DSB to half of the + * cycles served from MITE. + * - Coffeelake: Intel i9-9900k + * - Coffeelake: Intel i7-9700k + * + * I haven't been able to reproduce the instability or DSB misses on any + * of the following CPUS: + * - Haswell + * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH + * - Skylake + * + * Alignment is done for each of the three major decompression loops: + * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer + * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer + * - ZSTD_decompressSequences_body + * Alignment choices are made to minimize large swings on bad cases and influence on performance + * from changes external to this code, rather than to overoptimize on the current commit. + * + * If you are seeing performance stability this script can help test. + * It tests on 4 commits in zstd where I saw performance change. + * + * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 + */ #if defined(__GNUC__) && defined(__x86_64__) - /* Align the decompression loop to 32 + 16 bytes. - * - * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression - * speed swings based on the alignment of the decompression loop. This - * performance swing is caused by parts of the decompression loop falling - * out of the DSB. The entire decompression loop should fit in the DSB, - * when it can't we get much worse performance. You can measure if you've - * hit the good case or the bad case with this perf command for some - * compressed file test.zst: - * - * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ - * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst - * - * If you see most cycles served out of the MITE you've hit the bad case. - * If you see most cycles served out of the DSB you've hit the good case. - * If it is pretty even then you may be in an okay case. - * - * This issue has been reproduced on the following CPUs: - * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 - * Use Instruments->Counters to get DSB/MITE cycles. - * I never got performance swings, but I was able to - * go from the good case of mostly DSB to half of the - * cycles served from MITE. - * - Coffeelake: Intel i9-9900k - * - Coffeelake: Intel i7-9700k - * - * I haven't been able to reproduce the instability or DSB misses on any - * of the following CPUS: - * - Haswell - * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH - * - Skylake - * - * If you are seeing performance stability this script can help test. - * It tests on 4 commits in zstd where I saw performance change. - * - * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 - */ - __asm__(".p2align 6"); - __asm__("nop"); - __asm__(".p2align 5"); - __asm__("nop"); -# if __GNUC__ >= 9 && __GNUC__ < 11 - /* better for gcc-9 and gcc-10, worse for clang and gcc-8, gcc-11 */ - __asm__(".p2align 3"); + __asm__(".p2align 6"); +# if __GNUC__ >= 7 + /* good for gcc-7, gcc-9, and gcc-11 */ + __asm__("nop"); + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 4"); +# if __GNUC__ == 8 || __GNUC__ == 10 + /* good for gcc-8 and gcc-10 */ + __asm__("nop"); + __asm__(".p2align 3"); +# endif +# endif +#endif + + /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */ + for ( ; nbSeq; nbSeq--) { + sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + if (litPtr + sequence.litLength > dctx->litBufferEnd) break; + { size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } } + DEBUGLOG(6, "reached: (litPtr + sequence.litLength > dctx->litBufferEnd)"); + + /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */ + if (nbSeq > 0) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + DEBUGLOG(6, "There are %i sequences left, and %zu/%zu literals left in buffer", nbSeq, leftoverLit, sequence.litLength); + if (leftoverLit) { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence.litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } + nbSeq--; + } + } + + if (nbSeq > 0) { + /* there is remaining lit from extra buffer */ + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ != 7 + /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */ + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# elif __GNUC__ >= 11 + __asm__(".p2align 3"); # else - __asm__(".p2align 4"); + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); # endif #endif - for ( ; ; ) { - seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + + for ( ; nbSeq ; nbSeq--) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } + } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq); + RETURN_ERROR_IF(nbSeq, corruption_detected, ""); + DEBUGLOG(5, "bitStream : start=%p, ptr=%p, bitsConsumed=%u", seqState.DStream.start, seqState.DStream.ptr, seqState.DStream.bitsConsumed); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + if (dctx->litBufferLocation == ZSTD_split) { + /* split hasn't been reached yet, first get dst then copy litExtraBuffer */ + size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); + DEBUGLOG(6, "copy last literals from segment : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + } + /* copy last literals from internal buffer */ + { size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); + DEBUGLOG(6, "copy last literals from internal buffer : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } } + + DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); + return (size_t)(op - ostart); +} + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ZSTD_maybeNullPtrAdd(ostart, maxDstSize) : dctx->litBuffer; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); + const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq); + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ >= 7 + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# else + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; nbSeq ; nbSeq--) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif if (UNLIKELY(ZSTD_isError(oneSeqSize))) return oneSeqSize; DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); op += oneSeqSize; - if (UNLIKELY(!--nbSeq)) - break; - BIT_reloadDStream(&(seqState.DStream)); } /* check if reached exact end */ - DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); - RETURN_ERROR_IF(nbSeq, corruption_detected, ""); - RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + assert(nbSeq == 0); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); /* save reps for next block */ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } } /* last literal segment */ - { size_t const lastLLSize = litEnd - litPtr; + { size_t const lastLLSize = (size_t)(litEnd - litPtr); + DEBUGLOG(6, "copy last literals : %u", (U32)lastLLSize); RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); if (op != NULL) { ZSTD_memcpy(op, litPtr, lastLLSize); op += lastLLSize; - } - } + } } - return op-ostart; + DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); + return (size_t)(op - ostart); } static size_t ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} + +static size_t +ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT -FORCE_INLINE_TEMPLATE size_t -ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence, +FORCE_INLINE_TEMPLATE + +size_t ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence, const BYTE* const prefixStart, const BYTE* const dictEnd) { prefetchPos += sequence.litLength; { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart; - const BYTE* const match = matchBase + prefetchPos - sequence.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. - * No consequence though : memory address is only used for prefetching, not for dereferencing */ + /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : memory address is only used for prefetching, not for dereferencing */ + const BYTE* const match = ZSTD_wrappedPtrSub(ZSTD_wrappedPtrAdd(matchBase, prefetchPos), sequence.offset); PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ } return prefetchPos + sequence.matchLength; @@ -1189,20 +1735,18 @@ ZSTD_decompressSequencesLong_body( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { const BYTE* ip = (const BYTE*)seqStart; const BYTE* const iend = ip + seqSize; BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + maxDstSize; + BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ZSTD_maybeNullPtrAdd(ostart, maxDstSize); BYTE* op = ostart; const BYTE* litPtr = dctx->litPtr; - const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* litBufferEnd = dctx->litBufferEnd; const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - (void)frame; /* Regen sequences */ if (nbSeq) { @@ -1227,31 +1771,95 @@ ZSTD_decompressSequencesLong_body( ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); /* prepare in advance */ - for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNb<seqAdvance); seqNb++) { - seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + for (seqNb=0; seqNb<seqAdvance; seqNb++) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, seqNb == nbSeq-1); prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); sequences[seqNb] = sequence; } - RETURN_ERROR_IF(seqNb<seqAdvance, corruption_detected, ""); - /* decode and decompress */ - for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb<nbSeq) ; seqNb++) { - seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd); - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + /* decompress without stomping litBuffer */ + for (; seqNb < nbSeq; seqNb++) { + seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset, seqNb == nbSeq-1); - prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); - sequences[seqNb & STORED_SEQS_MASK] = sequence; - op += oneSeqSize; + if (dctx->litBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) { + /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */ + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } } + else + { + /* lit buffer is either wholly contained in first or second split, or not split at all*/ + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } } - RETURN_ERROR_IF(seqNb<nbSeq, corruption_detected, ""); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); /* finish queue */ seqNb -= seqAdvance; for ( ; seqNb<nbSeq ; seqNb++) { - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[seqNb&STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd); - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - op += oneSeqSize; + seq_t *sequence = &(sequences[seqNb&STORED_SEQS_MASK]); + if (dctx->litBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence->litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + else + { + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } } /* save reps for next block */ @@ -1259,25 +1867,34 @@ ZSTD_decompressSequencesLong_body( } /* last literal segment */ - { size_t const lastLLSize = litEnd - litPtr; + if (dctx->litBufferLocation == ZSTD_split) { /* first deplete literal buffer in dst, then copy litExtraBuffer */ + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + } + { size_t const lastLLSize = litBufferEnd - litPtr; RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); if (op != NULL) { - ZSTD_memcpy(op, litPtr, lastLLSize); + ZSTD_memmove(op, litPtr, lastLLSize); op += lastLLSize; } } - return op-ostart; + return (size_t)(op - ostart); } static size_t ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ @@ -1286,27 +1903,34 @@ ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, #if DYNAMIC_BMI2 #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG -static TARGET_ATTRIBUTE("bmi2") size_t +static BMI2_TARGET_ATTRIBUTE size_t DONT_VECTORIZE ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT -static TARGET_ATTRIBUTE("bmi2") size_t +static BMI2_TARGET_ATTRIBUTE size_t ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ @@ -1316,23 +1940,34 @@ typedef size_t (*ZSTD_decompressSequences_t)( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame); + const ZSTD_longOffset_e isLongOffset); #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG static size_t ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequences"); #if DYNAMIC_BMI2 if (ZSTD_DCtx_get_bmi2(dctx)) { - return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +static size_t +ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif - return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ @@ -1347,69 +1982,114 @@ static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset, - const int frame) + const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequencesLong"); #if DYNAMIC_BMI2 if (ZSTD_DCtx_get_bmi2(dctx)) { - return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif - return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ +/** + * @returns The total size of the history referenceable by zstd, including + * both the prefix and the extDict. At @p op any offset larger than this + * is invalid. + */ +static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart) +{ + return (size_t)(op - virtualStart); +} -#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) -/* ZSTD_getLongOffsetsShare() : +typedef struct { + unsigned longOffsetShare; + unsigned maxNbAdditionalBits; +} ZSTD_OffsetInfo; + +/* ZSTD_getOffsetInfo() : * condition : offTable must be valid * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) - * compared to maximum possible of (1<<OffFSELog) */ -static unsigned -ZSTD_getLongOffsetsShare(const ZSTD_seqSymbol* offTable) + * compared to maximum possible of (1<<OffFSELog), + * as well as the maximum number additional bits required. + */ +static ZSTD_OffsetInfo +ZSTD_getOffsetInfo(const ZSTD_seqSymbol* offTable, int nbSeq) { - const void* ptr = offTable; - U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog; - const ZSTD_seqSymbol* table = offTable + 1; - U32 const max = 1 << tableLog; - U32 u, total = 0; - DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog); - - assert(max <= (1 << OffFSELog)); /* max not too large */ - for (u=0; u<max; u++) { - if (table[u].nbAdditionalBits > 22) total += 1; + ZSTD_OffsetInfo info = {0, 0}; + /* If nbSeq == 0, then the offTable is uninitialized, but we have + * no sequences, so both values should be 0. + */ + if (nbSeq != 0) { + const void* ptr = offTable; + U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog; + const ZSTD_seqSymbol* table = offTable + 1; + U32 const max = 1 << tableLog; + U32 u; + DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog); + + assert(max <= (1 << OffFSELog)); /* max not too large */ + for (u=0; u<max; u++) { + info.maxNbAdditionalBits = MAX(info.maxNbAdditionalBits, table[u].nbAdditionalBits); + if (table[u].nbAdditionalBits > 22) info.longOffsetShare += 1; + } + + assert(tableLog <= OffFSELog); + info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */ } - assert(tableLog <= OffFSELog); - total <<= (OffFSELog - tableLog); /* scale to OffFSELog */ + return info; +} - return total; +/** + * @returns The maximum offset we can decode in one read of our bitstream, without + * reloading more bits in the middle of the offset bits read. Any offsets larger + * than this must use the long offset decoder. + */ +static size_t ZSTD_maxShortOffset(void) +{ + if (MEM_64bits()) { + /* We can decode any offset without reloading bits. + * This might change if the max window size grows. + */ + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + return (size_t)-1; + } else { + /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1. + * This offBase would require STREAM_ACCUMULATOR_MIN extra bits. + * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset. + */ + size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1; + size_t const maxOffset = maxOffbase - ZSTD_REP_NUM; + assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN); + return maxOffset; + } } -#endif size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, const int frame) + const void* src, size_t srcSize, const streaming_operation streaming) { /* blockType == blockCompressed */ const BYTE* ip = (const BYTE*)src; - /* isLongOffset must be true if there are long offsets. - * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN. - * We don't expect that to be the case in 64-bit mode. - * In block mode, window size is not known, so we have to be conservative. - * (note: but it could be evaluated from current-lowLimit) - */ - ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)))); - DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); - - RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); + DEBUGLOG(5, "ZSTD_decompressBlock_internal (cSize : %u)", (unsigned)srcSize); + + /* Note : the wording of the specification + * allows compressed block to be sized exactly ZSTD_blockSizeMax(dctx). + * This generally does not happen, as it makes little sense, + * since an uncompressed block would feature same size and have no decompression cost. + * Also, note that decoder from reference libzstd before < v1.5.4 + * would consider this edge case as an error. + * As a consequence, avoid generating compressed blocks of size ZSTD_blockSizeMax(dctx) + * for broader compatibility with the deployed ecosystem of zstd decoders */ + RETURN_ERROR_IF(srcSize > ZSTD_blockSizeMax(dctx), srcSize_wrong, ""); /* Decode literals section */ - { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); - DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize); if (ZSTD_isError(litCSize)) return litCSize; ip += litCSize; srcSize -= litCSize; @@ -1417,16 +2097,35 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, /* Build Decoding Tables */ { + /* Compute the maximum block size, which must also work when !frame and fParams are unset. + * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t. + */ + size_t const blockSizeMax = MIN(dstCapacity, ZSTD_blockSizeMax(dctx)); + size_t const totalHistorySize = ZSTD_totalHistorySize(ZSTD_maybeNullPtrAdd((BYTE*)dst, blockSizeMax), (BYTE const*)dctx->virtualStart); + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than ZSTD_maxShortOffset(). + * We don't expect that to be the case in 64-bit mode. + * + * We check here to see if our history is large enough to allow long offsets. + * If it isn't, then we can't possible have (valid) long offsets. If the offset + * is invalid, then it is okay to read it incorrectly. + * + * If isLongOffsets is true, then we will later check our decoding table to see + * if it is even possible to generate long offsets. + */ + ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset())); /* These macros control at build-time which decompressor implementation * we use. If neither is defined, we do some inspection and dispatch at * runtime. */ #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - int usePrefetchDecoder = 0; -#if ZSTD_DECOMPRESS_DICTIONARY != 0 - usePrefetchDecoder = dctx->ddictIsCold; -#endif + int usePrefetchDecoder = dctx->ddictIsCold; +#else + /* Set to 1 to avoid computing offset info if we don't need to. + * Otherwise this value is ignored. + */ + int usePrefetchDecoder = 1; #endif int nbSeq; size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); @@ -1434,38 +2133,55 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, ip += seqHSize; srcSize -= seqHSize; - RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall, + "invalid dst"); -#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - if ( !usePrefetchDecoder - && (!frame || (dctx->fParams.windowSize > (1<<24))) - && (nbSeq>ADVANCED_SEQS) ) { /* could probably use a larger nbSeq limit */ - U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); - U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ - usePrefetchDecoder = (shareLongOffsets >= minShare); + /* If we could potentially have long offsets, or we might want to use the prefetch decoder, + * compute information about the share of long offsets, and the maximum nbAdditionalBits. + * NOTE: could probably use a larger nbSeq limit + */ + if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) { + ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq); + if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) { + /* If isLongOffset, but the maximum number of additional bits that we see in our table is small + * enough, then we know it is impossible to have too long an offset in this block, so we can + * use the regular offset decoder. + */ + isLongOffset = ZSTD_lo_isRegularOffset; + } + if (!usePrefetchDecoder) { + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (info.longOffsetShare >= minShare); + } } -#endif -#if ZSTD_DECOMPRESS_DICTIONARY != 0 + dctx->ddictIsCold = 0; -#endif #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - if (usePrefetchDecoder) + if (usePrefetchDecoder) { +#else + (void)usePrefetchDecoder; + { #endif #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT - return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); #endif + } #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG /* else */ - return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); + if (dctx->litBufferLocation == ZSTD_split) + return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); + else + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); #endif } } +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) { if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ @@ -1477,13 +2193,24 @@ void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) } -size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { size_t dSize; + dctx->isFrameDecompression = 0; ZSTD_checkContinuity(dctx, dst, dstCapacity); - dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, not_streaming); + FORWARD_IF_ERROR(dSize, ""); dctx->previousDstEnd = (char*)dst + dSize; return dSize; } + + +/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */ +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize); +} diff --git a/src/bled/zstd_decompress_block.h b/src/bled/zstd_decompress_block.h index 47653414508..08261243d89 100644 --- a/src/bled/zstd_decompress_block.h +++ b/src/bled/zstd_decompress_block.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -33,6 +33,12 @@ */ + /* Streaming state is used to inform allocation of the literal buffer */ +typedef enum { + not_streaming = 0, + is_streaming = 1 +} streaming_operation; + /* ZSTD_decompressBlock_internal() : * decompress block, starting at `src`, * into destination buffer `dst`. @@ -41,7 +47,7 @@ */ size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, const int frame); + const void* src, size_t srcSize, const streaming_operation streaming); /* ZSTD_buildFSETable() : * generate FSE decoding table for one symbol (ll, ml or off) @@ -54,9 +60,14 @@ size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, */ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U32* nbAdditionalBits, + const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize, int bmi2); +/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */ +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + #endif /* ZSTD_DEC_BLOCK_H */ diff --git a/src/bled/zstd_decompress_internal.h b/src/bled/zstd_decompress_internal.h index 19841485580..6abf502f830 100644 --- a/src/bled/zstd_decompress_internal.h +++ b/src/bled/zstd_decompress_internal.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -20,7 +20,7 @@ * Dependencies *********************************************************/ #include "zstd_mem.h" /* BYTE, U16, U32 */ -#include "zstd_internal.h" /* ZSTD_seqSymbol */ +#include "zstd_internal.h" /* constants : MaxLL, MaxML, MaxOff, LLFSELog, etc. */ @@ -40,7 +40,7 @@ static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; -static UNUSED_ATTR const U32 OF_bits[MaxOff+1] = { +static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, @@ -75,12 +75,13 @@ static UNUSED_ATTR const U32 ML_base[MaxML+1] = { #define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) #define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) +#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12 typedef struct { ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ - HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */ U32 rep[ZSTD_REP_NUM]; U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; } ZSTD_entropyDTables_t; @@ -106,6 +107,22 @@ typedef struct { size_t ddictPtrCount; } ZSTD_DDictHashSet; +#ifndef ZSTD_DECODER_INTERNAL_BUFFER +# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16) +#endif + +#define ZSTD_LBMIN 64 +#define ZSTD_LBMAX (128 << 10) + +/* extra buffer, compensates when dst is not large enough to store litBuffer */ +#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX) + +typedef enum { + ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */ + ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */ + ZSTD_split = 2 /* Split between litExtraBuffer and dst */ +} ZSTD_litLocation_e; + struct ZSTD_DCtx_s { const ZSTD_seqSymbol* LLTptr; @@ -132,13 +149,15 @@ struct ZSTD_DCtx_s ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */ U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */ const BYTE* litPtr; + ZSTD_customMem customMem; size_t litSize; size_t rleSize; + size_t staticSize; + int isFrameDecompression; #if DYNAMIC_BMI2 != 0 int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ #endif -#if ZSTD_DECOMPRESS_DICTIONARY != 0 /* dictionary */ ZSTD_DDict* ddictLocal; const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ @@ -147,7 +166,8 @@ struct ZSTD_DCtx_s ZSTD_dictUses_e dictUses; ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ -#endif + int disableHufAsm; + int maxBlockSizeParam; /* streaming */ ZSTD_dStreamStage streamStage; @@ -160,18 +180,34 @@ struct ZSTD_DCtx_s size_t outStart; size_t outEnd; size_t lhSize; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; +#endif U32 hostageByte; int noForwardProgress; + ZSTD_bufferMode_e outBufferMode; ZSTD_outBuffer expectedOutBuffer; /* workspace */ - BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; + BYTE* litBuffer; + const BYTE* litBufferEnd; + ZSTD_litLocation_e litBufferLocation; + BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t oversizedDuration; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + void const* dictContentBeginForFuzzing; + void const* dictContentEndForFuzzing; +#endif /* Tracing */ +#if ZSTD_TRACE + ZSTD_TraceCtx traceCtx; +#endif }; /* typedef'd to ZSTD_DCtx within "zstd.h" */ MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { @@ -182,8 +218,7 @@ MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { return 0; #endif } -#define ZSTD_DCtx_get_staticSize(dctx) (0) -#define ZSTD_DCtx_get_outBufferMode(dctx) (ZSTD_bm_buffered) + /*-******************************************************* * Shared internal functions *********************************************************/ diff --git a/src/bled/zstd_deps.h b/src/bled/zstd_deps.h index 8a62920edf0..ce6cca4f100 100644 --- a/src/bled/zstd_deps.h +++ b/src/bled/zstd_deps.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -24,7 +24,18 @@ #ifndef ZSTD_DEPS_COMMON #define ZSTD_DEPS_COMMON -#include <assert.h> +/* Even though we use qsort_r only for the dictionary builder, the macro + * _GNU_SOURCE has to be declared *before* the inclusion of any standard + * header and the script 'combine.sh' combines the whole zstd source code + * in a single file. + */ +#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \ + defined(__CYGWIN__) || defined(__MSYS__) +#if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */ +#define _GNU_SOURCE +#endif +#endif + #include <limits.h> #include <stddef.h> #include <string.h> @@ -85,6 +96,13 @@ #include <assert.h> #endif /* ZSTD_DEPS_ASSERT */ + +#else + +#ifndef assert +#define assert(condition) ((void)0) +#endif + #endif /* ZSTD_DEPS_NEED_ASSERT */ /* Need: @@ -114,17 +132,13 @@ #endif /* ZSTD_DEPS_NEED_STDINT */ #define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1]) + #include "libbb.h" + +#if (DEBUGLEVEL >= 1) #define RAWLOG(l, ...) { if (l <= ZSTD_DEBUGLEVEL) bb_printf(__VA_ARGS__); } #define DEBUGLOG(l, ...) { if (l <= ZSTD_DEBUGLEVEL) bb_printf(__VA_ARGS__); } - -#if defined(__GNUC__) -# define MEM_STATIC static __inline __attribute__((unused)) -#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define MEM_STATIC static inline -#elif defined(_MSC_VER) -# define MEM_STATIC static __inline #else -# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#define RAWLOG(l, ...) do { } while (0) +#define DEBUGLOG(l, ...) do { } while (0) #endif - diff --git a/src/bled/zstd_entropy_common.c b/src/bled/zstd_entropy_common.c index 5103f0f0f28..e66e7ee2436 100644 --- a/src/bled/zstd_entropy_common.c +++ b/src/bled/zstd_entropy_common.c @@ -1,6 +1,6 @@ /* ****************************************************************** * Common functions of New Generation Entropy library - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -19,8 +19,8 @@ #include "zstd_error_private.h" /* ERR_*, ERROR */ #define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ #include "fse.h" -#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ #include "huf.h" +#include "zstd_bits.h" /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */ /*=== Version ===*/ @@ -28,38 +28,16 @@ unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } /*=== Error Management ===*/ -#define FSE_isError ERR_isError +unsigned FSE_isError(size_t code) { return ERR_isError(code); } const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } -#define HUF_isError ERR_isError +unsigned HUF_isError(size_t code) { return ERR_isError(code); } const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } /*-************************************************************** * FSE NCount encoding-decoding ****************************************************************/ -static U32 FSE_ctz(U32 val) -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ - unsigned long r=0; - return _BitScanForward(&r, val) ? (unsigned)r : 0; -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return __builtin_ctz(val); -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return __CTZ(val); -# else /* Software version */ - U32 count = 0; - while ((val & 1) == 0) { - val >>= 1; - ++count; - } - return count; -# endif - } -} - FORCE_INLINE_TEMPLATE size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize) @@ -107,7 +85,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne * repeat. * Avoid UB by setting the high bit to 1. */ - int repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + int repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; while (repeats >= 12) { charnum += 3 * 12; if (LIKELY(ip <= iend-7)) { @@ -118,7 +96,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne ip = iend - 4; } bitStream = MEM_readLE32(ip) >> bitCount; - repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; } charnum += 3 * repeats; bitStream >>= 2 * repeats; @@ -183,7 +161,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne * know that threshold > 1. */ if (remaining <= 1) break; - nbBits = BIT_highbit32(remaining) + 1; + nbBits = ZSTD_highbit32(remaining) + 1; threshold = 1 << (nbBits - 1); } if (charnum >= maxSV1) break; @@ -217,7 +195,7 @@ static size_t FSE_readNCount_body_default( } #if DYNAMIC_BMI2 -TARGET_ATTRIBUTE("bmi2") static size_t FSE_readNCount_body_bmi2( +BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2( short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize) { @@ -258,7 +236,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, const void* src, size_t srcSize) { U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; - return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0); + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0); } FORCE_INLINE_TEMPLATE size_t @@ -306,14 +284,14 @@ HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ - { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + { U32 const tableLog = ZSTD_highbit32(weightTotal) + 1; if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); *tableLogPtr = tableLog; /* determine last weight */ { U32 const total = 1 << tableLog; U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; + U32 const verif = 1 << ZSTD_highbit32(rest); + U32 const lastWeight = ZSTD_highbit32(rest) + 1; if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ huffWeight[oSize] = (BYTE)lastWeight; rankStats[lastWeight]++; @@ -337,7 +315,7 @@ static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* r } #if DYNAMIC_BMI2 -static TARGET_ATTRIBUTE("bmi2") size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, +static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) @@ -350,13 +328,13 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, - int bmi2) + int flags) { #if DYNAMIC_BMI2 - if (bmi2) { + if (flags & HUF_flags_bmi2) { return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } #endif - (void)bmi2; + (void)flags; return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } diff --git a/src/bled/zstd_error_private.c b/src/bled/zstd_error_private.c index 531d1ee7b49..23b571dd937 100644 --- a/src/bled/zstd_error_private.c +++ b/src/bled/zstd_error_private.c @@ -1,5 +1,5 @@ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -27,9 +27,11 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(version_unsupported): return "Version not supported"; case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; - case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(corruption_detected): return "Data corruption detected"; case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification"; case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters"; case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; case PREFIX(init_missing): return "Context should be init first"; case PREFIX(memory_allocation): return "Allocation error : not enough memory"; @@ -38,17 +40,22 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected"; case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; case PREFIX(dictionary_wrong): return "Dictionary mismatch"; case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; case PREFIX(srcSize_wrong): return "Src size is incorrect"; case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; + case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full"; + case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty"; /* following error codes are not stable and may be removed or changed in a future version */ case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; + case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code"; + case PREFIX(externalSequences_invalid): return "External sequences are not valid"; case PREFIX(maxCode): default: return notErrorCode; } diff --git a/src/bled/zstd_error_private.h b/src/bled/zstd_error_private.h index fa75e32a01c..1463ff987a6 100644 --- a/src/bled/zstd_error_private.h +++ b/src/bled/zstd_error_private.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -13,6 +13,9 @@ #ifndef ERROR_H_MODULE #define ERROR_H_MODULE +#if defined (__cplusplus) +extern "C" { +#endif /* **************************************** @@ -28,7 +31,7 @@ ******************************************/ #if defined(__GNUC__) # define ERR_STATIC static __attribute__((unused)) -#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # define ERR_STATIC static inline #elif defined(_MSC_VER) # define ERR_STATIC static __inline @@ -56,8 +59,13 @@ ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } /* check and forward error code */ -#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e -#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } +#define CHECK_V_F(e, f) \ + size_t const e = f; \ + do { \ + if (ERR_isError(e)) \ + return e; \ + } while (0) +#define CHECK_F(f) do { CHECK_V_F(_var_err__, f); } while (0) /*-**************************************** @@ -68,12 +76,7 @@ const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ ERR_STATIC const char* ERR_getErrorName(size_t code) { -#ifdef ZSTD_STRIP_ERROR_STRINGS - (void)code; - return "Error strings stripped"; -#else return ERR_getErrorString(ERR_getErrorCode(code)); -#endif } /** @@ -96,10 +99,12 @@ void _force_has_format_string(const char *format, ...) { * We want to force this function invocation to be syntactically correct, but * we don't want to force runtime evaluation of its arguments. */ -#define _FORCE_HAS_FORMAT_STRING(...) \ - if (0) { \ - _force_has_format_string(__VA_ARGS__); \ - } +#define _FORCE_HAS_FORMAT_STRING(...) \ + do { \ + if (0) { \ + _force_has_format_string(__VA_ARGS__); \ + } \ + } while (0) #define ERR_QUOTE(str) #str @@ -110,48 +115,53 @@ void _force_has_format_string(const char *format, ...) { * In order to do that (particularly, printing the conditional that failed), * this can't just wrap RETURN_ERROR(). */ -#define RETURN_ERROR_IF(cond, err, ...) \ - if (cond) { \ - RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ - __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return ERROR(err); \ - } +#define RETURN_ERROR_IF(cond, err, ...) \ + do { \ + if (cond) { \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } \ + } while (0) /** * Unconditionally return the specified error. * * In debug modes, prints additional information. */ -#define RETURN_ERROR(err, ...) \ - do { \ - RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ - __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return ERROR(err); \ - } while(0); +#define RETURN_ERROR(err, ...) \ + do { \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } while(0) /** * If the provided expression evaluates to an error code, returns that error code. * * In debug modes, prints additional information. */ -#define FORWARD_IF_ERROR(err, ...) \ - do { \ - size_t const err_code = (err); \ - if (ERR_isError(err_code)) { \ - RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ - __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \ - _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ - RAWLOG(3, ": " __VA_ARGS__); \ - RAWLOG(3, "\n"); \ - return err_code; \ - } \ - } while(0); - +#define FORWARD_IF_ERROR(err, ...) \ + do { \ + size_t const err_code = (err); \ + if (ERR_isError(err_code)) { \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ + __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return err_code; \ + } \ + } while(0) + +#if defined (__cplusplus) +} +#endif #endif /* ERROR_H_MODULE */ diff --git a/src/bled/zstd_errors.h b/src/bled/zstd_errors.h index 576d46dfb65..20e488f1551 100644 --- a/src/bled/zstd_errors.h +++ b/src/bled/zstd_errors.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -11,14 +11,37 @@ #ifndef ZSTD_ERRORS_H_398273423 #define ZSTD_ERRORS_H_398273423 +#if defined (__cplusplus) +extern "C" { +#endif -/*===== dependency =====*/ -#include "zstd_deps.h" +/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDERRORLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDERRORLIB_VISIBLE +# endif +#endif +#ifndef ZSTDERRORLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDERRORLIB_HIDDEN +# endif +#endif -/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ -#define ZSTDERRORLIB_VISIBILITY MEM_STATIC -#define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE +#endif /*-********************************************* * Error codes list @@ -43,14 +66,17 @@ typedef enum { ZSTD_error_frameParameter_windowTooLarge = 16, ZSTD_error_corruption_detected = 20, ZSTD_error_checksum_wrong = 22, + ZSTD_error_literals_headerWrong = 24, ZSTD_error_dictionary_corrupted = 30, ZSTD_error_dictionary_wrong = 32, ZSTD_error_dictionaryCreation_failed = 34, ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_combination_unsupported = 41, ZSTD_error_parameter_outOfBound = 42, ZSTD_error_tableLog_tooLarge = 44, ZSTD_error_maxSymbolValue_tooLarge = 46, ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_stabilityCondition_notRespected = 50, ZSTD_error_stage_wrong = 60, ZSTD_error_init_missing = 62, ZSTD_error_memory_allocation = 64, @@ -58,20 +84,23 @@ typedef enum { ZSTD_error_dstSize_tooSmall = 70, ZSTD_error_srcSize_wrong = 72, ZSTD_error_dstBuffer_null = 74, + ZSTD_error_noForwardProgress_destFull = 80, + ZSTD_error_noForwardProgress_inputEmpty = 82, /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ ZSTD_error_frameIndex_tooLarge = 100, ZSTD_error_seekableIO = 102, ZSTD_error_dstBuffer_wrong = 104, ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_sequenceProducer_failed = 106, + ZSTD_error_externalSequences_invalid = 107, ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ } ZSTD_ErrorCode; -/*! ZSTD_getErrorCode() : - convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, - which can be used to compare with enum list published above */ -ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ +#if defined (__cplusplus) +} +#endif #endif /* ZSTD_ERRORS_H_398273423 */ diff --git a/src/bled/zstd_internal.h b/src/bled/zstd_internal.h index 549a259e3ea..40d1436a7db 100644 --- a/src/bled/zstd_internal.h +++ b/src/bled/zstd_internal.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -20,17 +20,27 @@ * Dependencies ***************************************/ #include "zstd_compiler.h" +#include "zstd_cpu.h" #include "zstd_mem.h" #include "zstd_error_private.h" #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" #define FSE_STATIC_LINKING_ONLY #include "fse.h" -#define HUF_STATIC_LINKING_ONLY #include "huf.h" +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif #include "xxhash.h" /* XXH_reset, update, digest */ -#define ZSTD_TRACE 0 +#ifndef ZSTD_NO_TRACE +# include "zstd_trace.h" +#else +# define ZSTD_TRACE 0 +#endif +#if defined (__cplusplus) +extern "C" { +#endif /* ---- static assert (debug) --- */ #define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) @@ -46,6 +56,7 @@ #undef MAX #define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define BOUNDED(min,val,max) (MAX(min,MIN(val,max))) /*-************************************* @@ -54,7 +65,6 @@ #define ZSTD_OPT_NUM (1<<12) #define ZSTD_REP_NUM 3 /* number of repcodes */ -#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; #define KB *(1 <<10) @@ -81,9 +91,9 @@ typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; #define ZSTD_FRAMECHECKSUMSIZE 4 #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ -#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */ +#define MIN_LITERALS_FOR_4_STREAMS 6 -#define HufLog 12 typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; #define LONGNBSEQ 0x7F00 @@ -91,6 +101,7 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy #define MINMATCH 3 #define Litbits 8 +#define LitHufLog 11 #define MaxLit ((1<<Litbits) - 1) #define MaxML 52 #define MaxLL 35 @@ -101,12 +112,14 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy #define LLFSELog 9 #define OffFSELog 8 #define MaxFSELog MAX(MAX(MLFSELog, LLFSELog), OffFSELog) +#define MaxMLBits 16 +#define MaxLLBits 16 #define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */ /* Each table cannot take more than #symbols * FSELog bits */ #define ZSTD_MAX_FSE_HEADERS_SIZE (((MaxML + 1) * MLFSELog + (MaxLL + 1) * LLFSELog + (MaxOff + 1) * OffFSELog + 7) / 8) -static UNUSED_ATTR const U32 LL_bits[MaxLL+1] = { +static UNUSED_ATTR const U8 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, @@ -123,7 +136,7 @@ static UNUSED_ATTR const S16 LL_defaultNorm[MaxLL+1] = { #define LL_DEFAULTNORMLOG 6 /* for static allocation */ static UNUSED_ATTR const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG; -static UNUSED_ATTR const U32 ML_bits[MaxML+1] = { +static UNUSED_ATTR const U8 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -164,16 +177,27 @@ static void ZSTD_copy8(void* dst, const void* src) { ZSTD_memcpy(dst, src, 8); #endif } +#define COPY8(d,s) do { ZSTD_copy8(d,s); d+=8; s+=8; } while (0) -#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } +/* Need to use memmove here since the literal buffer can now be located within + the dst buffer. In circumstances where the op "catches up" to where the + literal buffer is, there can be partial overlaps in this call on the final + copy if the literal is being shifted by less than 16 bytes. */ static void ZSTD_copy16(void* dst, const void* src) { #if defined(ZSTD_ARCH_ARM_NEON) vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); +#elif defined(ZSTD_ARCH_X86_SSE2) + _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); +#elif defined(__clang__) + ZSTD_memmove(dst, src, 16); #else - ZSTD_memcpy(dst, src, 16); + /* ZSTD_memmove is not inlined properly by gcc */ + BYTE copy16_buf[16]; + ZSTD_memcpy(copy16_buf, src, 16); + ZSTD_memcpy(dst, copy16_buf, 16); #endif } -#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } +#define COPY16(d,s) do { ZSTD_copy16(d,s); d+=16; s+=16; } while (0) #define WILDCOPY_OVERLENGTH 32 #define WILDCOPY_VECLEN 16 @@ -199,12 +223,10 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e BYTE* op = (BYTE*)dst; BYTE* const oend = op + length; - assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff <= -WILDCOPY_VECLEN)); - if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) { /* Handle short offset copies. */ do { - COPY8(op, ip) + COPY8(op, ip); } while (op < oend); } else { assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); @@ -214,12 +236,6 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e * one COPY16() in the first call. Then, do two calls per loop since * at that point it is more likely to have a high trip count. */ -#ifdef __aarch64__ - do { - COPY16(op, ip); - } - while (op < oend); -#else ZSTD_copy16(op, ip); if (16 >= length) return; op += 16; @@ -229,7 +245,6 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e COPY16(op, ip); } while (op < oend); -#endif } } @@ -263,9 +278,9 @@ typedef enum { * Private declarations *********************************************/ typedef struct seqDef_s { - U32 offset; /* offset == rawOffset + ZSTD_REP_NUM, or equivalently, offCode + 1 */ + U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */ U16 litLength; - U16 matchLength; + U16 mlBase; /* mlBase == matchLength - MINMATCH */ } seqDef; /* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */ @@ -278,11 +293,11 @@ typedef enum { typedef struct { seqDef* sequencesStart; seqDef* sequences; /* ptr to end of sequences */ - BYTE* litStart; - BYTE* lit; /* ptr to end of literals */ - BYTE* llCode; - BYTE* mlCode; - BYTE* ofCode; + BYTE* litStart; + BYTE* lit; /* ptr to end of literals */ + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; size_t maxNbSeq; size_t maxNbLit; @@ -290,8 +305,8 @@ typedef struct { * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment * the existing value of the litLength or matchLength by 0x10000. */ - ZSTD_longLengthType_e longLengthType; - U32 longLengthPos; /* Index of the sequence to apply long length modification to */ + ZSTD_longLengthType_e longLengthType; + U32 longLengthPos; /* Index of the sequence to apply long length modification to */ } seqStore_t; typedef struct { @@ -307,13 +322,13 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore { ZSTD_sequenceLength seqLen; seqLen.litLength = seq->litLength; - seqLen.matchLength = seq->matchLength + MINMATCH; + seqLen.matchLength = seq->mlBase + MINMATCH; if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { if (seqStore->longLengthType == ZSTD_llt_literalLength) { - seqLen.litLength += 0xFFFF; + seqLen.litLength += 0x10000; } if (seqStore->longLengthType == ZSTD_llt_matchLength) { - seqLen.matchLength += 0xFFFF; + seqLen.matchLength += 0x10000; } } return seqLen; @@ -326,85 +341,13 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` */ typedef struct { + size_t nbBlocks; size_t compressedSize; unsigned long long decompressedBound; } ZSTD_frameSizeInfo; /* decompress & legacy */ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ -void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ - -MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ -# if STATIC_BMI2 == 1 - return _lzcnt_u32(val)^31; -# else - unsigned long r=0; - return _BitScanReverse(&r, val) ? (unsigned)r : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return __builtin_clz (val) ^ 31; -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return 31 - __CLZ(val); -# else /* Software version */ - static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[(v * 0x07C4ACDDU) >> 27]; -# endif - } -} - -/** - * Computes CTZ on a U64. - * This will be slow on 32-bit mode, and on unsupported compilers. - * If you need this function to be fast (because it is hot) expand - * support. - */ -MEM_STATIC unsigned ZSTD_countTrailingZeros(size_t val) -{ - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) -# if STATIC_BMI2 - return _tzcnt_u64(val); -# else - unsigned long r = 0; - return _BitScanForward64( &r, (U64)val ) ? (unsigned)(r >> 3) : 0; -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 4) - return __builtin_ctzll((U64)val); -# else - static const int DeBruijnBytePos[64] = { 0, 1, 2, 7, 3, 13, 8, 19, - 4, 25, 14, 28, 9, 34, 20, 56, - 5, 17, 26, 54, 15, 41, 29, 43, - 10, 31, 38, 35, 21, 45, 49, 57, - 63, 6, 12, 18, 24, 27, 33, 55, - 16, 53, 40, 42, 30, 37, 44, 48, - 62, 11, 23, 32, 52, 39, 36, 47, - 61, 22, 51, 46, 60, 50, 59, 58 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else { /* 32 bits */ -# if defined(_MSC_VER) - unsigned long r=0; - return _BitScanForward( &r, (U32)val ) ? (unsigned)(r >> 3) : 0; -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 1, 28, 2, 29, 14, 24, 3, - 30, 22, 20, 15, 25, 17, 4, 8, - 31, 27, 13, 23, 21, 19, 16, 7, - 26, 12, 18, 6, 11, 5, 10, 9 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } -} +int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ /* ZSTD_invalidateRepCodes() : @@ -422,28 +365,27 @@ typedef struct { /*! ZSTD_getcBlockSize() : * Provides the size of compressed block from block header `src` */ -/* Used by: decompress, fullbench (does not get its definition from here) */ +/* Used by: decompress, fullbench */ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr); /*! ZSTD_decodeSeqHeaders() : * decode sequence header from src */ -/* Used by: decompress, fullbench (does not get its definition from here) */ +/* Used by: zstd_decompress_block, fullbench */ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, const void* src, size_t srcSize); -/* - * Custom memory allocation functions +/** + * @returns true iff the CPU supports dynamic BMI2 dispatch. */ -typedef struct { void* customAlloc; void* customFree; void* opaque; } ZSTD_customMem; -extern ZSTD_customMem ZSTD_defaultCMem; -static inline void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) { - (void)customMem; - return malloc(size); +MEM_STATIC int ZSTD_cpuSupportsBmi2(void) +{ + ZSTD_cpuid_t cpuid = ZSTD_cpuid(); + return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid); } -static inline void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) { - (void)customMem; - free(ptr); + +#if defined (__cplusplus) } +#endif #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/src/bled/zstd_mem.h b/src/bled/zstd_mem.h index b5d15f1abb6..5e94d813733 100644 --- a/src/bled/zstd_mem.h +++ b/src/bled/zstd_mem.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -11,13 +11,15 @@ #ifndef MEM_H_MODULE #define MEM_H_MODULE +#if defined (__cplusplus) +extern "C" { +#endif /*-**************************************** * Dependencies ******************************************/ #include <stddef.h> /* size_t, ptrdiff_t */ #include "zstd_compiler.h" /* __has_builtin */ -//#include "debug.h" /* DEBUG_STATIC_ASSERT */ #include "zstd_deps.h" /* ZSTD_memcpy */ @@ -27,6 +29,8 @@ #if defined(_MSC_VER) /* Visual Studio */ # include <stdlib.h> /* _byteswap_ulong */ # include <intrin.h> /* _byteswap_* */ +#elif defined(__ICCARM__) +# include <intrinsics.h> #endif /*-************************************************************** @@ -39,6 +43,8 @@ # include <stdint.h> /* intptr_t */ # endif typedef uint8_t BYTE; + typedef uint8_t U8; + typedef int8_t S8; typedef uint16_t U16; typedef int16_t S16; typedef uint32_t U32; @@ -51,6 +57,8 @@ # error "this implementation requires char to be exactly 8-bit type" #endif typedef unsigned char BYTE; + typedef unsigned char U8; + typedef signed char S8; #if USHRT_MAX != 65535 # error "this implementation requires short to be exactly 16-bit type" #endif @@ -117,21 +125,15 @@ MEM_STATIC size_t MEM_swapST(size_t in); /*-************************************************************** * Memory I/O Implementation *****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. +/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory: + * Method 0 : always use `memcpy()`. Safe and portable. + * Method 1 : Use compiler extension to set unaligned access. * Method 2 : direct access. This method is portable but violate C standard. * It can generate buggy code on targets depending on alignment. - * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) + * Default : method 1 if supported, else method 0 */ #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) +# ifdef __GNUC__ # define MEM_FORCE_MEMORY_ACCESS 1 # endif #endif @@ -153,6 +155,8 @@ MEM_STATIC unsigned MEM_isLittleEndian(void) return 1; #elif defined(__DMC__) && defined(_M_IX86) return 1; +#elif defined(__IAR_SYSTEMS_ICC__) && __LITTLE_ENDIAN__ + return 1; #else const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ return one.c[0]; @@ -174,30 +178,19 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) - __pragma( pack(push, 1) ) - typedef struct { U16 v; } unalign16; - typedef struct { U32 v; } unalign32; - typedef struct { U64 v; } unalign64; - typedef struct { size_t v; } unalignArch; - __pragma( pack(pop) ) -#else - typedef struct { U16 v; } __attribute__((packed)) unalign16; - typedef struct { U32 v; } __attribute__((packed)) unalign32; - typedef struct { U64 v; } __attribute__((packed)) unalign64; - typedef struct { size_t v; } __attribute__((packed)) unalignArch; -#endif +typedef __attribute__((aligned(1))) U16 unalign16; +typedef __attribute__((aligned(1))) U32 unalign32; +typedef __attribute__((aligned(1))) U64 unalign64; +typedef __attribute__((aligned(1))) size_t unalignArch; -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; } -MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; } +MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; } +MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; } -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; } +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; } #else @@ -241,6 +234,14 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) #endif /* MEM_FORCE_MEMORY_ACCESS */ +MEM_STATIC U32 MEM_swap32_fallback(U32 in) +{ + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +} + MEM_STATIC U32 MEM_swap32(U32 in) { #if defined(_MSC_VER) /* Visual Studio */ @@ -248,23 +249,16 @@ MEM_STATIC U32 MEM_swap32(U32 in) #elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ || (defined(__clang__) && __has_builtin(__builtin_bswap32)) return __builtin_bswap32(in); +#elif defined(__ICCARM__) + return __REV(in); #else - return ((in << 24) & 0xff000000 ) | - ((in << 8) & 0x00ff0000 ) | - ((in >> 8) & 0x0000ff00 ) | - ((in >> 24) & 0x000000ff ); + return MEM_swap32_fallback(in); #endif } -MEM_STATIC U64 MEM_swap64(U64 in) +MEM_STATIC U64 MEM_swap64_fallback(U64 in) { -#if defined(_MSC_VER) /* Visual Studio */ - return _byteswap_uint64(in); -#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ - || (defined(__clang__) && __has_builtin(__builtin_bswap64)) - return __builtin_bswap64(in); -#else - return ((in << 56) & 0xff00000000000000ULL) | + return ((in << 56) & 0xff00000000000000ULL) | ((in << 40) & 0x00ff000000000000ULL) | ((in << 24) & 0x0000ff0000000000ULL) | ((in << 8) & 0x000000ff00000000ULL) | @@ -272,6 +266,17 @@ MEM_STATIC U64 MEM_swap64(U64 in) ((in >> 24) & 0x0000000000ff0000ULL) | ((in >> 40) & 0x000000000000ff00ULL) | ((in >> 56) & 0x00000000000000ffULL); +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap64)) + return __builtin_bswap64(in); +#else + return MEM_swap64_fallback(in); #endif } @@ -418,6 +423,8 @@ MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) /* code only tested on 32 and 64 bits systems */ MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } - +#if defined (__cplusplus) +} +#endif #endif /* MEM_H_MODULE */ diff --git a/src/rufus.rc b/src/rufus.rc index 59fd437a314..da4048bdeae 100644 --- a/src/rufus.rc +++ b/src/rufus.rc @@ -33,7 +33,7 @@ LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL IDD_DIALOG DIALOGEX 12, 12, 232, 326 STYLE DS_SETFONT | DS_MODALFRAME | DS_CENTER | WS_MINIMIZEBOX | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_ACCEPTFILES -CAPTION "Rufus 4.7.2211" +CAPTION "Rufus 4.7.2212" FONT 9, "Segoe UI Symbol", 400, 0, 0x0 BEGIN LTEXT "Drive Properties",IDS_DRIVE_PROPERTIES_TXT,8,6,53,12,NOT WS_GROUP @@ -399,8 +399,8 @@ END // VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,7,2211,0 - PRODUCTVERSION 4,7,2211,0 + FILEVERSION 4,7,2212,0 + PRODUCTVERSION 4,7,2212,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS 0x1L @@ -418,13 +418,13 @@ BEGIN VALUE "Comments", "https://rufus.ie" VALUE "CompanyName", "Akeo Consulting" VALUE "FileDescription", "Rufus" - VALUE "FileVersion", "4.7.2211" + VALUE "FileVersion", "4.7.2212" VALUE "InternalName", "Rufus" VALUE "LegalCopyright", " 2011-2024 Pete Batard (GPL v3)" VALUE "LegalTrademarks", "https://www.gnu.org/licenses/gpl-3.0.html" VALUE "OriginalFilename", "rufus-4.7.exe" VALUE "ProductName", "Rufus" - VALUE "ProductVersion", "4.7.2211" + VALUE "ProductVersion", "4.7.2212" END END BLOCK "VarFileInfo"