diff --git a/CompressSave/CompressSave.cs b/CompressSave/CompressSave.cs index 6d98b6f..8025926 100644 --- a/CompressSave/CompressSave.cs +++ b/CompressSave/CompressSave.cs @@ -51,8 +51,16 @@ public class CompressSave : BaseUnityPlugin new ConfigDescription("Set default compression type for manual saves.", new AcceptableValueList("lz4", "zstd", "none"), new { })) .Value); + PatchSave.CompressionTypeForAutoSaves = CompressionTypeFromString( + Config.Bind("Compression", "TypeForAuto", StringFromCompresstionType(PatchSave.CompressionTypeForSaves), + new ConfigDescription("Set default compression type for auto saves and last-exit save.", + new AcceptableValueList("lz4", "zstd", "none"), new { })) + .Value); PatchSave.CompressionLevelForSaves = Config.Bind("Compression", "Level", PatchSave.CompressionLevelForSaves, - "Set default compression level.\n0 for default level.\n3 ~ 12 for lz4, -5 ~ 22 for zstd.\nSmaller level leads to faster speed and less compression ratio.") + "Set default compression level for manual saves.\n0 for default level.\n3 ~ 12 for lz4, -5 ~ 22 for zstd.\nSmaller level leads to faster speed and less compression ratio.") + .Value; + PatchSave.CompressionLevelForAutoSaves = Config.Bind("Compression", "LevelForAuto", PatchSave.CompressionLevelForAutoSaves, + "Set default compression level for auto saves and last-exit save.\n0 for default level.\n3 ~ 12 for lz4, -5 ~ 22 for zstd.\nSmaller level leads to faster speed and less compression ratio.") .Value; PatchSave.CreateCompressBuffer(); if (GameConfig.gameVersion != SaveUtil.VerifiedVersion) @@ -88,45 +96,40 @@ public class CompressSave : BaseUnityPlugin class PatchSave { - public static readonly WrapperDefines LZ4Wrapper = new LZ4API(), ZstdWrapper = new ZstdAPI(), NoneWrapper = new NoneAPI(); - private const long SizeInMBytes = 1024 * 1024; + public static readonly WrapperDefines LZ4Wrapper = new LZ4API(), ZstdWrapper = new ZstdAPI(); + private static readonly WrapperDefines NoneWrapper = new NoneAPI(); private static CompressionStream.CompressBuffer _compressBuffer; public static bool UseCompressSave; private static CompressionType _compressionTypeForLoading = CompressionType.None; + private static CompressionType _compressionTypeForSaving = CompressionType.Zstd; + private static int _compressionLevelForSaving = 0; public static CompressionType CompressionTypeForSaves = CompressionType.Zstd; + public static CompressionType CompressionTypeForAutoSaves = CompressionType.Zstd; public static int CompressionLevelForSaves; + public static int CompressionLevelForAutoSaves; private static Stream _compressionStream; public static bool EnableCompress; public static void CreateCompressBuffer() { - switch (CompressionTypeForSaves) - { - case CompressionType.LZ4: - _compressBuffer = CompressionStream.CreateBuffer(LZ4Wrapper, (int)SizeInMBytes); - break; - case CompressionType.Zstd: - _compressBuffer = CompressionStream.CreateBuffer(ZstdWrapper, (int)SizeInMBytes); - break; - case CompressionType.None: - _compressBuffer = CompressionStream.CreateBuffer(NoneWrapper, (int)SizeInMBytes); - break; - default: - throw new ArgumentException("Unknown compression type."); - } + var bufSize = CompressionStream.MB; + var outBufSize = LZ4Wrapper.CompressBufferBound(bufSize); + outBufSize = Math.Max(outBufSize, ZstdWrapper.CompressBufferBound(bufSize)); + outBufSize = Math.Max(outBufSize, NoneWrapper.CompressBufferBound(bufSize)); + _compressBuffer = CompressionStream.CreateBuffer((int)outBufSize, bufSize); } private static void WriteHeader(FileStream fileStream) { - switch (CompressionTypeForSaves) + switch (_compressionTypeForSaving) { case CompressionType.Zstd: - for (int i = 0; i < 3; i++) + for (var i = 0; i < 3; i++) fileStream.WriteByte(0xCC); fileStream.WriteByte(0xCD); break; case CompressionType.LZ4: - for (int i = 0; i < 4; i++) + for (var i = 0; i < 4; i++) fileStream.WriteByte(0xCC); break; case CompressionType.None: @@ -142,6 +145,11 @@ class PatchSave static void BeforeAutoSave() { UseCompressSave = EnableCompress; + if (UseCompressSave) + { + _compressionTypeForSaving = CompressionTypeForAutoSaves; + _compressionLevelForSaving = CompressionLevelForAutoSaves; + } } [HarmonyTranspiler] @@ -200,13 +208,13 @@ class PatchSave { SaveUtil.logger.LogDebug("Begin compress save"); WriteHeader(fileStream); - switch (CompressionTypeForSaves) + switch (_compressionTypeForSaving) { case CompressionType.LZ4: - _compressionStream = new CompressionStream(LZ4Wrapper, CompressionLevelForSaves, fileStream, _compressBuffer, true); + _compressionStream = new CompressionStream(LZ4Wrapper, _compressionLevelForSaving, fileStream, _compressBuffer, true); break; case CompressionType.Zstd: - _compressionStream = new CompressionStream(ZstdWrapper, CompressionLevelForSaves, fileStream, _compressBuffer, true); + _compressionStream = new CompressionStream(ZstdWrapper, _compressionLevelForSaving, fileStream, _compressBuffer, true); break; case CompressionType.None: _compressionStream = new CompressionStream(NoneWrapper, 0, fileStream, _compressBuffer, true); @@ -247,7 +255,7 @@ class PatchSave } var writeflag = _compressionStream.CanWrite; Stream stream = null; - if (writeflag && CompressionTypeForSaves == CompressionType.None) + if (writeflag && _compressionTypeForSaving == CompressionType.None) { stream = ((CompressionStream)_compressionStream).outStream; } @@ -264,6 +272,8 @@ class PatchSave writer.Write(saveLen); writer.Dispose(); } + _compressionTypeForSaving = CompressionTypeForSaves; + _compressionLevelForSaving = CompressionLevelForSaves; UseCompressSave = false; } } diff --git a/CompressSave/CompressSave.csproj b/CompressSave/CompressSave.csproj index fdabd5e..e77fe09 100644 --- a/CompressSave/CompressSave.csproj +++ b/CompressSave/CompressSave.csproj @@ -4,7 +4,7 @@ CompressSave org.soardev.compresssave DSP MOD - CompressSave - 1.2.2 + 1.3.0 true latest net472 diff --git a/CompressSave/LZ4WrapC/lz4/LICENSE b/CompressSave/LZ4WrapC/lz4/LICENSE new file mode 100644 index 0000000..4884916 --- /dev/null +++ b/CompressSave/LZ4WrapC/lz4/LICENSE @@ -0,0 +1,24 @@ +LZ4 Library +Copyright (c) 2011-2020, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/CompressSave/LZ4WrapC/lz4/README.md b/CompressSave/LZ4WrapC/lz4/README.md new file mode 100644 index 0000000..a86cd0d --- /dev/null +++ b/CompressSave/LZ4WrapC/lz4/README.md @@ -0,0 +1,187 @@ +LZ4 - Library Files +================================ + +The `/lib` directory contains many files, but depending on project's objectives, +not all of them are required. +Limited systems may want to reduce the nb of source files to include +as a way to reduce binary size and dependencies. + +Capabilities are added at the "level" granularity, detailed below. + +#### Level 1 : Minimal LZ4 build + +The minimum required is **`lz4.c`** and **`lz4.h`**, +which provides the fast compression and decompression algorithms. +They generate and decode data using the [LZ4 block format]. + + +#### Level 2 : High Compression variant + +For more compression ratio at the cost of compression speed, +the High Compression variant called **lz4hc** is available. +Add files **`lz4hc.c`** and **`lz4hc.h`**. +This variant also compresses data using the [LZ4 block format], +and depends on regular `lib/lz4.*` source files. + + +#### Level 3 : Frame support, for interoperability + +In order to produce compressed data compatible with `lz4` command line utility, +it's necessary to use the [official interoperable frame format]. +This format is generated and decoded automatically by the **lz4frame** library. +Its public API is described in `lib/lz4frame.h`. +In order to work properly, lz4frame needs all other modules present in `/lib`, +including, lz4 and lz4hc, and also **xxhash**. +So it's necessary to also include `xxhash.c` and `xxhash.h`. + + +#### Level 4 : File compression operations + +As a helper around file operations, +the library has been recently extended with `lz4file.c` and `lz4file.h` +(still considered experimental at the time of this writing). +These helpers allow opening, reading, writing, and closing files +using transparent LZ4 compression / decompression. +As a consequence, using `lz4file` adds a dependency on ``. + +`lz4file` relies on `lz4frame` in order to produce compressed data +conformant to the [LZ4 Frame format] specification. +Consequently, to enable this capability, +it's necessary to include all `*.c` and `*.h` files from `lib/` directory. + + +#### Advanced / Experimental API + +Definitions which are not guaranteed to remain stable in future versions, +are protected behind macros, such as `LZ4_STATIC_LINKING_ONLY`. +As the name suggests, these definitions should only be invoked +in the context of static linking ***only***. +Otherwise, dependent application may fail on API or ABI break in the future. +The associated symbols are also not exposed by the dynamic library by default. +Should they be nonetheless needed, it's possible to force their publication +by using build macros `LZ4_PUBLISH_STATIC_FUNCTIONS` +and `LZ4F_PUBLISH_STATIC_FUNCTIONS`. + + +#### Build macros + +The following build macro can be selected to adjust source code behavior at compilation time : + +- `LZ4_FAST_DEC_LOOP` : this triggers a speed optimized decompression loop, more powerful on modern cpus. + This loop works great on `x86`, `x64` and `aarch64` cpus, and is automatically enabled for them. + It's also possible to enable or disable it manually, by passing `LZ4_FAST_DEC_LOOP=1` or `0` to the preprocessor. + For example, with `gcc` : `-DLZ4_FAST_DEC_LOOP=1`, + and with `make` : `CPPFLAGS+=-DLZ4_FAST_DEC_LOOP=1 make lz4`. + +- `LZ4_DISTANCE_MAX` : control the maximum offset that the compressor will allow. + Set to 65535 by default, which is the maximum value supported by lz4 format. + Reducing maximum distance will reduce opportunities for LZ4 to find matches, + hence will produce a worse compression ratio. + Setting a smaller max distance could allow compatibility with specific decoders with limited memory budget. + This build macro only influences the compressed output of the compressor. + +- `LZ4_DISABLE_DEPRECATE_WARNINGS` : invoking a deprecated function will make the compiler generate a warning. + This is meant to invite users to update their source code. + Should this be a problem, it's generally possible to make the compiler ignore these warnings, + for example with `-Wno-deprecated-declarations` on `gcc`, + or `_CRT_SECURE_NO_WARNINGS` for Visual Studio. + This build macro offers another project-specific method + by defining `LZ4_DISABLE_DEPRECATE_WARNINGS` before including the LZ4 header files. + +- `LZ4_FORCE_SW_BITCOUNT` : by default, the compression algorithm tries to determine lengths + by using bitcount instructions, generally implemented as fast single instructions in many cpus. + In case the target cpus doesn't support it, or compiler intrinsic doesn't work, or feature bad performance, + it's possible to use an optimized software path instead. + This is achieved by setting this build macros. + In most cases, it's not expected to be necessary, + but it can be legitimately considered for less common platforms. + +- `LZ4_ALIGN_TEST` : alignment test ensures that the memory area + passed as argument to become a compression state is suitably aligned. + This test can be disabled if it proves flaky, by setting this value to 0. + +- `LZ4_USER_MEMORY_FUNCTIONS` : replace calls to ``'s `malloc()`, `calloc()` and `free()` + by user-defined functions, which must be named `LZ4_malloc()`, `LZ4_calloc()` and `LZ4_free()`. + User functions must be available at link time. + +- `LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION` : + Remove support of dynamic memory allocation. + For more details, see description of this macro in `lib/lz4.c`. + +- `LZ4_FREESTANDING` : by setting this build macro to 1, + LZ4/HC removes dependencies on the C standard library, + including allocation functions and `memmove()`, `memcpy()`, and `memset()`. + This build macro is designed to help use LZ4/HC in restricted environments + (embedded, bootloader, etc). + For more details, see description of this macro in `lib/lz4.h`. + +- `LZ4_HEAPMODE` : Select how stateless compression functions like `LZ4_compress_default()` + allocate memory for their hash table, + in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). + +- `LZ4HC_HEAPMODE` : Select how stateless HC compression functions like `LZ4_compress_HC()` + allocate memory for their workspace: + in stack (0), or in heap (1:default). + Since workspace is rather large, stack can be inconvenient, hence heap mode is recommended. + +- `LZ4F_HEAPMODE` : selects how `LZ4F_compressFrame()` allocates the compression state, + either on stack (default, value 0) or using heap memory (value 1). + + +#### Makefile variables + +The following `Makefile` variables can be selected to alter the profile of produced binaries : +- `BUILD_SHARED` : generate `libzstd` dynamic library (enabled by default) +- `BUILD_STATIC` : generate `libzstd` static library (enabled by default) + + +#### Amalgamation + +lz4 source code can be amalgamated into a single file. +One can combine all source code into `lz4_all.c` by using following command: +``` +cat lz4.c lz4hc.c lz4frame.c > lz4_all.c +``` +(`cat` file order is important) then compile `lz4_all.c`. +All `*.h` files present in `/lib` remain necessary to compile `lz4_all.c`. + + +#### Windows : using MinGW+MSYS to create DLL + +DLL can be created using MinGW+MSYS with the `make liblz4` command. +This command creates `dll\liblz4.dll` and the import library `dll\liblz4.lib`. +To override the `dlltool` command when cross-compiling on Linux, just set the `DLLTOOL` variable. Example of cross compilation on Linux with mingw-w64 64 bits: +``` +make BUILD_STATIC=no CC=x86_64-w64-mingw32-gcc DLLTOOL=x86_64-w64-mingw32-dlltool OS=Windows_NT +``` +The import library is only required with Visual C++. +The header files `lz4.h`, `lz4hc.h`, `lz4frame.h` and the dynamic library +`dll\liblz4.dll` are required to compile a project using gcc/MinGW. +The dynamic library has to be added to linking options. +It means that if a project that uses LZ4 consists of a single `test-dll.c` +file it should be linked with `dll\liblz4.dll`. For example: +``` + $(CC) $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll +``` +The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`. + + +#### Miscellaneous + +Other files present in the directory are not source code. They are : + + - `LICENSE` : contains the BSD license text + - `Makefile` : `make` script to compile and install lz4 library (static and dynamic) + - `liblz4.pc.in` : for `pkg-config` (used in `make install`) + - `README.md` : this file + +[official interoperable frame format]: ../doc/lz4_Frame_format.md +[LZ4 Frame format]: ../doc/lz4_Frame_format.md +[LZ4 block format]: ../doc/lz4_Block_format.md + + +#### License + +All source material within __lib__ directory are BSD 2-Clause licensed. +See [LICENSE](LICENSE) for details. +The license is also reminded at the top of each source file. diff --git a/CompressSave/LZ4WrapC/lz4/lz4.c b/CompressSave/LZ4WrapC/lz4/lz4.c index 8999137..0982f95 100644 --- a/CompressSave/LZ4WrapC/lz4/lz4.c +++ b/CompressSave/LZ4WrapC/lz4/lz4.c @@ -37,7 +37,8 @@ **************************************/ /* * LZ4_HEAPMODE : - * Select how default compression functions will allocate memory for their hash table, + * Select how stateless compression functions like `LZ4_compress_default()` + * allocate memory for their hash table, * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). */ #ifndef LZ4_HEAPMODE @@ -279,7 +280,7 @@ static const int LZ4_minLength = (MFLIMIT+1); static int g_debuglog_enable = 1; # define DEBUGLOG(l, ...) { \ if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ - fprintf(stderr, __FILE__ ": "); \ + fprintf(stderr, __FILE__ " %i: ", __LINE__); \ fprintf(stderr, __VA_ARGS__); \ fprintf(stderr, " \n"); \ } } @@ -379,14 +380,16 @@ static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ /* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) LZ4_unalign; +typedef struct { U16 u16; } __attribute__((packed)) LZ4_unalign16; +typedef struct { U32 u32; } __attribute__((packed)) LZ4_unalign32; +typedef struct { reg_t uArch; } __attribute__((packed)) LZ4_unalignST; -static U16 LZ4_read16(const void* ptr) { return ((const LZ4_unalign*)ptr)->u16; } -static U32 LZ4_read32(const void* ptr) { return ((const LZ4_unalign*)ptr)->u32; } -static reg_t LZ4_read_ARCH(const void* ptr) { return ((const LZ4_unalign*)ptr)->uArch; } +static U16 LZ4_read16(const void* ptr) { return ((const LZ4_unalign16*)ptr)->u16; } +static U32 LZ4_read32(const void* ptr) { return ((const LZ4_unalign32*)ptr)->u32; } +static reg_t LZ4_read_ARCH(const void* ptr) { return ((const LZ4_unalignST*)ptr)->uArch; } -static void LZ4_write16(void* memPtr, U16 value) { ((LZ4_unalign*)memPtr)->u16 = value; } -static void LZ4_write32(void* memPtr, U32 value) { ((LZ4_unalign*)memPtr)->u32 = value; } +static void LZ4_write16(void* memPtr, U16 value) { ((LZ4_unalign16*)memPtr)->u16 = value; } +static void LZ4_write32(void* memPtr, U32 value) { ((LZ4_unalign32*)memPtr)->u32 = value; } #else /* safe and portable access using memcpy() */ @@ -524,12 +527,12 @@ LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const si case 2: LZ4_memcpy(v, srcPtr, 2); LZ4_memcpy(&v[2], srcPtr, 2); -#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */ +#if defined(_MSC_VER) && (_MSC_VER <= 1936) /* MSVC 2022 ver 17.6 or earlier */ # pragma warning(push) # pragma warning(disable : 6385) /* warning C6385: Reading invalid data from 'v'. */ #endif LZ4_memcpy(&v[4], v, 4); -#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */ +#if defined(_MSC_VER) && (_MSC_VER <= 1936) /* MSVC 2022 ver 17.6 or earlier */ # pragma warning(pop) #endif break; @@ -803,23 +806,19 @@ LZ4_FORCE_INLINE void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableT } } +/* LZ4_putPosition*() : only used in byPtr mode */ LZ4_FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, - void* tableBase, tableType_t const tableType, - const BYTE* srcBase) + void* tableBase, tableType_t const tableType) { - switch (tableType) - { - case clearedTable: { /* illegal! */ assert(0); return; } - case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } - case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } - case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } - } + const BYTE** const hashTable = (const BYTE**)tableBase; + assert(tableType == byPtr); (void)tableType; + hashTable[h] = p; } -LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType) { U32 const h = LZ4_hashPosition(p, tableType); - LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); + LZ4_putPositionOnHash(p, h, tableBase, tableType); } /* LZ4_getIndexOnHash() : @@ -901,7 +900,7 @@ LZ4_prepareTable(LZ4_stream_t_internal* const cctx, /** LZ4_compress_generic() : * inlined, to ensure branches are decided at compilation time. - * Presumed already validated at this stage: + * The following conditions are presumed already validated: * - source != NULL * - inputSize > 0 */ @@ -919,10 +918,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( const int acceleration) { int result; - const BYTE* ip = (const BYTE*) source; + const BYTE* ip = (const BYTE*)source; U32 const startIndex = cctx->currentOffset; - const BYTE* base = (const BYTE*) source - startIndex; + const BYTE* base = (const BYTE*)source - startIndex; const BYTE* lowLimit; const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx; @@ -930,7 +929,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary; const U32 dictSize = dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize; - const U32 dictDelta = (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with index in current context */ + const U32 dictDelta = + (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with indexes in current context */ int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ @@ -955,11 +955,11 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( DEBUGLOG(5, "LZ4_compress_generic_validated: srcSize=%i, tableType=%u", inputSize, tableType); assert(ip != NULL); + if (tableType == byU16) assert(inputSize=LZ4_64Klimit)) { return 0; } /* Size too large (not within 64K limit) */ - if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ assert(acceleration >= 1); lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); @@ -979,7 +979,12 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( if (inputSizehashTable, tableType, base); + { U32 const h = LZ4_hashPosition(ip, tableType); + if (tableType == byPtr) { + LZ4_putPositionOnHash(ip, h, cctx->hashTable, byPtr); + } else { + LZ4_putIndexOnHash(startIndex, h, cctx->hashTable, tableType); + } } ip++; forwardH = LZ4_hashPosition(ip, tableType); /* Main Loop */ @@ -1004,7 +1009,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType); forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType); } while ( (match+LZ4_DISTANCE_MAX < ip) || (LZ4_read32(match) != LZ4_read32(ip)) ); @@ -1202,13 +1207,19 @@ _next_match: if (ip >= mflimitPlusOne) break; /* Fill table */ - LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); + { U32 const h = LZ4_hashPosition(ip-2, tableType); + if (tableType == byPtr) { + LZ4_putPositionOnHash(ip-2, h, cctx->hashTable, byPtr); + } else { + U32 const idx = (U32)((ip-2) - base); + LZ4_putIndexOnHash(idx, h, cctx->hashTable, tableType); + } } /* Test next position */ if (tableType == byPtr) { match = LZ4_getPosition(ip, cctx->hashTable, tableType); - LZ4_putPosition(ip, cctx->hashTable, tableType, base); + LZ4_putPosition(ip, cctx->hashTable, tableType); if ( (match+LZ4_DISTANCE_MAX >= ip) && (LZ4_read32(match) == LZ4_read32(ip)) ) { token=op++; *token=0; goto _next_match; } @@ -1222,6 +1233,7 @@ _next_match: if (dictDirective == usingDictCtx) { if (matchIndex < startIndex) { /* there was no match, try the dictionary */ + assert(tableType == byU32); matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); match = dictBase + matchIndex; lowLimit = dictionary; /* required for match length counter */ @@ -1375,9 +1387,10 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int */ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) { - LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; + LZ4_stream_t_internal* const ctx = &((LZ4_stream_t*)state)->internal_donotuse; if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + assert(ctx != NULL); if (dstCapacity >= LZ4_compressBound(srcSize)) { if (srcSize < LZ4_64Klimit) { @@ -1411,17 +1424,17 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst } -int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +int LZ4_compress_fast(const char* src, char* dest, int srcSize, int dstCapacity, int acceleration) { int result; #if (LZ4_HEAPMODE) - LZ4_stream_t* ctxPtr = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + LZ4_stream_t* const ctxPtr = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ if (ctxPtr == NULL) return 0; #else LZ4_stream_t ctx; LZ4_stream_t* const ctxPtr = &ctx; #endif - result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); + result = LZ4_compress_fast_extState(ctxPtr, src, dest, srcSize, dstCapacity, acceleration); #if (LZ4_HEAPMODE) FREEMEM(ctxPtr); @@ -1430,9 +1443,9 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp } -int LZ4_compress_default(const char* src, char* dst, int srcSize, int maxOutputSize) +int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity) { - return LZ4_compress_fast(src, dst, srcSize, maxOutputSize, 1); + return LZ4_compress_fast(src, dst, srcSize, dstCapacity, 1); } @@ -1459,11 +1472,11 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) { #if (LZ4_HEAPMODE) - LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + LZ4_stream_t* const ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ if (ctx == NULL) return 0; #else LZ4_stream_t ctxBody; - LZ4_stream_t* ctx = &ctxBody; + LZ4_stream_t* const ctx = &ctxBody; #endif int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); @@ -1538,11 +1551,11 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream) #define HASH_UNIT sizeof(reg_t) int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) { - LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; const tableType_t tableType = byU32; const BYTE* p = (const BYTE*)dictionary; const BYTE* const dictEnd = p + dictSize; - const BYTE* base; + U32 idx32; DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict); @@ -1565,14 +1578,15 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) } if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; - base = dictEnd - dict->currentOffset; dict->dictionary = p; dict->dictSize = (U32)(dictEnd - p); dict->tableType = (U32)tableType; + idx32 = dict->currentOffset - dict->dictSize; while (p <= dictEnd-HASH_UNIT) { - LZ4_putPosition(p, dict->hashTable, tableType, base); - p+=3; + U32 const h = LZ4_hashPosition(p, tableType); + LZ4_putIndexOnHash(idx32, h, dict->hashTable, tableType); + p+=3; idx32+=3; } return (int)dict->dictSize; @@ -1709,7 +1723,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, /* Hidden debug function, to force-test external dictionary mode */ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize) { - LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; + LZ4_stream_t_internal* const streamPtr = &LZ4_dict->internal_donotuse; int result; LZ4_renormDictT(streamPtr, srcSize); @@ -1772,7 +1786,7 @@ typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; * does not know end of input * presumes input is well formed * note : will consume at least one byte */ -size_t read_long_length_no_check(const BYTE** pp) +static size_t read_long_length_no_check(const BYTE** pp) { size_t b, l = 0; do { b = **pp; (*pp)++; l += b; } while (b==255); @@ -1991,6 +2005,7 @@ LZ4_decompress_generic( } /* Fast loop : decode sequences as long as output < oend-FASTLOOP_SAFE_DISTANCE */ + DEBUGLOG(6, "using fast decode loop"); while (1) { /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ assert(oend - op >= FASTLOOP_SAFE_DISTANCE); @@ -2001,7 +2016,10 @@ LZ4_decompress_generic( /* decode literal length */ if (length == RUN_MASK) { size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); - if (addl == rvl_error) { goto _output_error; } + if (addl == rvl_error) { + DEBUGLOG(6, "error reading long literal length"); + goto _output_error; + } length += addl; if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ @@ -2024,6 +2042,7 @@ LZ4_decompress_generic( /* get offset */ offset = LZ4_readLE16(ip); ip+=2; + DEBUGLOG(6, " offset = %zu", offset); match = op - offset; assert(match <= op); /* overflow check */ @@ -2032,11 +2051,17 @@ LZ4_decompress_generic( if (length == ML_MASK) { size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); - if (addl == rvl_error) { goto _output_error; } + if (addl == rvl_error) { + DEBUGLOG(6, "error reading long match length"); + goto _output_error; + } length += addl; length += MINMATCH; if (unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { + DEBUGLOG(6, "Error : offset outside buffers"); + goto _output_error; + } if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { goto safe_match_copy; } @@ -2060,7 +2085,10 @@ LZ4_decompress_generic( continue; } } } - if (checkOffset && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ + if ( checkOffset && (unlikely(match + dictSize < lowPrefix)) ) { + DEBUGLOG(6, "Error : pos=%zi, offset=%zi => outside buffers", op-lowPrefix, op-match); + goto _output_error; + } /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { assert(dictEnd != NULL); @@ -2069,7 +2097,8 @@ LZ4_decompress_generic( DEBUGLOG(7, "partialDecoding: dictionary match, close to dstEnd"); length = MIN(length, (size_t)(oend-op)); } else { - goto _output_error; /* end-of-block condition violated */ + DEBUGLOG(6, "end-of-block condition violated") + goto _output_error; } } if (length <= (size_t)(lowPrefix-match)) { @@ -2109,6 +2138,7 @@ LZ4_decompress_generic( #endif /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ + DEBUGLOG(6, "using safe decode loop"); while (1) { assert(ip < iend); token = *ip++; @@ -2416,6 +2446,7 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const void* dictStart, size_t dictSize) { + DEBUGLOG(5, "LZ4_decompress_safe_forceExtDict"); return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, decode_full_block, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); diff --git a/CompressSave/LZ4WrapC/lz4/lz4.h b/CompressSave/LZ4WrapC/lz4/lz4.h index 491c608..f85b038 100644 --- a/CompressSave/LZ4WrapC/lz4/lz4.h +++ b/CompressSave/LZ4WrapC/lz4/lz4.h @@ -189,8 +189,9 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); /*! LZ4_decompress_safe() : - * compressedSize : is the exact complete size of the compressed block. - * dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size. + * @compressedSize : is the exact complete size of the compressed block. + * @dstCapacity : is the size of destination buffer (which must be already allocated), + * is an upper bound of decompressed size. * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) * If destination buffer is not large enough, decoding will stop and output an error code (negative value). * If the source stream is detected malformed, the function will stop decoding and return a negative result. @@ -255,7 +256,7 @@ LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* d * @return : Nb bytes written into 'dst' (necessarily <= targetDestSize) * or 0 if compression fails. * - * Note : from v1.8.2 to v1.9.1, this function had a bug (fixed un v1.9.2+): + * Note : from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+): * the produced compressed content could, in specific circumstances, * require to be decompressed into a destination buffer larger * by at least 1 byte than the content to decompress. @@ -443,11 +444,24 @@ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); #define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ -/*! LZ4_decompress_*_continue() : - * These decoding functions allow decompression of consecutive blocks in "streaming" mode. - * A block is an unsplittable entity, it must be presented entirely to a decompression function. - * Decompression functions only accepts one block at a time. - * The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded. +/*! LZ4_decompress_safe_continue() : + * This decoding function allows decompression of consecutive blocks in "streaming" mode. + * The difference with the usual independent blocks is that + * new blocks are allowed to find references into former blocks. + * A block is an unsplittable entity, and must be presented entirely to the decompression function. + * LZ4_decompress_safe_continue() only accepts one block at a time. + * It's modeled after `LZ4_decompress_safe()` and behaves similarly. + * + * @LZ4_streamDecode : decompression state, tracking the position in memory of past data + * @compressedSize : exact complete size of one compressed block. + * @dstCapacity : size of destination buffer (which must be already allocated), + * must be an upper bound of decompressed size. + * @return : number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * + * The last 64KB of previously decoded data *must* remain available and unmodified + * at the memory position where they were previously decoded. * If less than 64KB of data has been decoded, all the data must be present. * * Special : if decompression side sets a ring buffer, it must respect one of the following conditions : @@ -474,10 +488,10 @@ LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, int srcSize, int dstCapacity); -/*! LZ4_decompress_*_usingDict() : - * These decoding functions work the same as - * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() - * They are stand-alone, and don't need an LZ4_streamDecode_t structure. +/*! LZ4_decompress_safe_usingDict() : + * Works the same as + * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_safe_continue() + * However, it's stateless: it doesn't need any LZ4_streamDecode_t state. * Dictionary is presumed stable : it must remain accessible and unmodified during decompression. * Performance tip : Decompression speed can be substantially increased * when dst == dictStart + dictSize. @@ -487,6 +501,12 @@ LZ4_decompress_safe_usingDict(const char* src, char* dst, int srcSize, int dstCapacity, const char* dictStart, int dictSize); +/*! LZ4_decompress_safe_partial_usingDict() : + * Behaves the same as LZ4_decompress_safe_partial() + * with the added ability to specify a memory segment for past data. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. + */ LZ4LIB_API int LZ4_decompress_safe_partial_usingDict(const char* src, char* dst, int compressedSize, diff --git a/CompressSave/LZ4WrapC/lz4/lz4frame.c b/CompressSave/LZ4WrapC/lz4/lz4frame.c index 4d991e1..77857a5 100644 --- a/CompressSave/LZ4WrapC/lz4/lz4frame.c +++ b/CompressSave/LZ4WrapC/lz4/lz4frame.c @@ -54,7 +54,7 @@ **************************************/ /* * LZ4F_HEAPMODE : - * Select how LZ4F_compressFrame will allocate the Compression Context, + * Control how LZ4F_compressFrame allocates the Compression State, * either on stack (0:default, fastest), or in memory heap (1:requires malloc()). */ #ifndef LZ4F_HEAPMODE @@ -314,9 +314,9 @@ static LZ4F_errorCode_t LZ4F_returnErrorCode(LZ4F_errorCodes code) #define RETURN_ERROR(e) return LZ4F_returnErrorCode(LZ4F_ERROR_ ## e) -#define RETURN_ERROR_IF(c,e) if (c) RETURN_ERROR(e) +#define RETURN_ERROR_IF(c,e) do { if (c) RETURN_ERROR(e); } while (0) -#define FORWARD_IF_ERROR(r) if (LZ4F_isError(r)) return (r) +#define FORWARD_IF_ERROR(r) do { if (LZ4F_isError(r)) return (r); } while (0) unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } @@ -1085,7 +1085,6 @@ size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) { - RETURN_ERROR_IF(cctxPtr->prefs.frameInfo.blockMode != LZ4F_blockIndependent, blockMode_invalid); return LZ4F_compressUpdateImpl(cctxPtr, dstBuffer, dstCapacity, srcBuffer, srcSize, diff --git a/CompressSave/LZ4WrapC/lz4/lz4frame.h b/CompressSave/LZ4WrapC/lz4/lz4frame.h index 8d9380b..e8ce66d 100644 --- a/CompressSave/LZ4WrapC/lz4/lz4frame.h +++ b/CompressSave/LZ4WrapC/lz4/lz4frame.h @@ -215,9 +215,19 @@ LZ4FLIB_API int LZ4F_compressionLevel_max(void); /* v1.8.0+ */ LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); /*! LZ4F_compressFrame() : - * Compress an entire srcBuffer into a valid LZ4 frame. - * dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). - * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. + * Compress srcBuffer content into an LZ4-compressed frame. + * It's a one shot operation, all input content is consumed, and all output is generated. + * + * Note : it's a stateless operation (no LZ4F_cctx state needed). + * In order to reduce load on the allocator, LZ4F_compressFrame(), by default, + * uses the stack to allocate space for the compression state and some table. + * If this usage of the stack is too much for your application, + * consider compiling `lz4frame.c` with compile-time macro LZ4F_HEAPMODE set to 1 instead. + * All state allocations will use the Heap. + * It also means each invocation of LZ4F_compressFrame() will trigger several internal alloc/free invocations. + * + * @dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * @preferencesPtr is optional : one can provide NULL, in which case all preferences are set to default. * @return : number of bytes written into dstBuffer. * or an error code if it fails (can be tested using LZ4F_isError()) */ @@ -463,6 +473,11 @@ LZ4F_getFrameInfo(LZ4F_dctx* dctx, * `dstBuffer` can freely change between each consecutive function invocation. * `dstBuffer` content will be overwritten. * + * Note: if `LZ4F_getFrameInfo()` is called before `LZ4F_decompress()`, srcBuffer must be updated to reflect + * the number of bytes consumed after reading the frame header. Failure to update srcBuffer before calling + * `LZ4F_decompress()` will cause decompression failure or, even worse, successful but incorrect decompression. + * See the `LZ4F_getFrameInfo()` docs for details. + * * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. * Schematically, it's the size of the current (or remaining) compressed block + header of next block. * Respecting the hint provides some small speed benefit, because it skips intermediate buffers. @@ -660,7 +675,7 @@ LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr, const LZ4F_decompressOptions_t* decompressOptionsPtr); -/*! Custom memory allocation : +/*! Custom memory allocation : v1.9.4+ * These prototypes make it possible to pass custom allocation/free functions. * LZ4F_customMem is provided at state creation time, using LZ4F_create*_advanced() listed below. * All allocation/free operations will be completed using these custom variants instead of regular ones. diff --git a/CompressSave/LZ4WrapC/lz4/lz4hc.c b/CompressSave/LZ4WrapC/lz4/lz4hc.c index e83246b..651f190 100644 --- a/CompressSave/LZ4WrapC/lz4/lz4hc.c +++ b/CompressSave/LZ4WrapC/lz4/lz4hc.c @@ -39,9 +39,10 @@ ***************************************/ /*! HEAPMODE : - * Select how default compression function will allocate workplace memory, - * in stack (0:fastest), or in heap (1:requires malloc()). - * Since workplace is rather large, heap mode is recommended. + * Select how stateless HC compression functions like `LZ4_compress_HC()` + * allocate memory for their workspace: + * in stack (0:fastest), or in heap (1:default, requires malloc()). + * Since workspace is rather large, heap mode is recommended. **/ #ifndef LZ4HC_HEAPMODE # define LZ4HC_HEAPMODE 1 @@ -102,6 +103,7 @@ static void LZ4HC_init_internal (LZ4HC_CCtx_internal* hc4, const BYTE* start) { size_t const bufferSize = (size_t)(hc4->end - hc4->prefixStart); size_t newStartingOffset = bufferSize + hc4->dictLimit; + DEBUGLOG(5, "LZ4HC_init_internal"); assert(newStartingOffset >= bufferSize); /* check overflow */ if (newStartingOffset > 1 GB) { LZ4HC_clearTables(hc4); @@ -237,13 +239,17 @@ static int LZ4HC_protectDictEnd(U32 const dictLimit, U32 const matchIndex) typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e; typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e; -LZ4_FORCE_INLINE int +typedef struct { + int off; + int len; +} LZ4HC_match_t; + +LZ4_FORCE_INLINE LZ4HC_match_t LZ4HC_InsertAndGetWiderMatch ( LZ4HC_CCtx_internal* const hc4, const BYTE* const ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, int longest, - const BYTE** matchpos, const BYTE** startpos, const int maxNbAttempts, const int patternAnalysis, const int chainSwap, @@ -251,7 +257,7 @@ LZ4HC_InsertAndGetWiderMatch ( const HCfavor_e favorDecSpeed) { U16* const chainTable = hc4->chainTable; - U32* const HashTable = hc4->hashTable; + U32* const hashTable = hc4->hashTable; const LZ4HC_CCtx_internal * const dictCtx = hc4->dictCtx; const BYTE* const prefixPtr = hc4->prefixStart; const U32 prefixIdx = hc4->dictLimit; @@ -268,22 +274,26 @@ LZ4HC_InsertAndGetWiderMatch ( U32 matchIndex; repeat_state_e repeat = rep_untested; size_t srcPatternLength = 0; + int offset = 0; DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch"); + assert(startpos != NULL); + *startpos = ip; /* in case there is no solution */ /* First Match */ - LZ4HC_Insert(hc4, ip); - matchIndex = HashTable[LZ4HC_hashPtr(ip)]; - DEBUGLOG(7, "First match at index %u / %u (lowestMatchIndex)", - matchIndex, lowestMatchIndex); + LZ4HC_Insert(hc4, ip); /* insert all prior positions up to ip (excluded) */ + matchIndex = hashTable[LZ4HC_hashPtr(ip)]; + DEBUGLOG(7, "First candidate match for pos %u found at index %u / %u (lowestMatchIndex)", + ipIndex, matchIndex, lowestMatchIndex); while ((matchIndex>=lowestMatchIndex) && (nbAttempts>0)) { int matchLength=0; nbAttempts--; assert(matchIndex < ipIndex); if (favorDecSpeed && (ipIndex - matchIndex < 8)) { - /* do nothing */ + /* do nothing: + * favorDecSpeed intentionally skips matches with offset < 8 */ } else if (matchIndex >= prefixIdx) { /* within current Prefix */ - const BYTE* const matchPtr = prefixPtr + matchIndex - prefixIdx; + const BYTE* const matchPtr = prefixPtr + (matchIndex - prefixIdx); assert(matchPtr < ip); assert(longest >= 1); if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) { @@ -293,10 +303,11 @@ LZ4HC_InsertAndGetWiderMatch ( matchLength -= back; if (matchLength > longest) { longest = matchLength; - *matchpos = matchPtr + back; + offset = (int)(ipIndex - matchIndex); *startpos = ip + back; + DEBUGLOG(7, "Found match of len=%i within prefix, offset=%i, back=%i", longest, offset, -back); } } } - } else { /* lowestMatchIndex <= matchIndex < dictLimit */ + } else { /* lowestMatchIndex <= matchIndex < dictLimit : within Ext Dict */ const BYTE* const matchPtr = dictStart + (matchIndex - dictIdx); assert(matchIndex >= dictIdx); if ( likely(matchIndex <= prefixIdx - 4) @@ -311,8 +322,9 @@ LZ4HC_InsertAndGetWiderMatch ( matchLength -= back; if (matchLength > longest) { longest = matchLength; - *matchpos = prefixPtr - prefixIdx + matchIndex + back; /* virtual pos, relative to ip, to retrieve offset */ + offset = (int)(ipIndex - matchIndex); *startpos = ip + back; + DEBUGLOG(7, "Found match of len=%i within dict, offset=%i, back=%i", longest, offset, -back); } } } if (chainSwap && matchLength==longest) { /* better match => select a better chain */ @@ -345,6 +357,7 @@ LZ4HC_InsertAndGetWiderMatch ( if (repeat == rep_untested) { if ( ((pattern & 0xFFFF) == (pattern >> 16)) & ((pattern & 0xFF) == (pattern >> 24)) ) { + DEBUGLOG(7, "Repeat pattern detected, char %02X", pattern >> 24); repeat = rep_confirmed; srcPatternLength = LZ4HC_countPattern(ip+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern); } else { @@ -353,7 +366,7 @@ LZ4HC_InsertAndGetWiderMatch ( if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex) && LZ4HC_protectDictEnd(prefixIdx, matchCandidateIdx) ) { const int extDict = matchCandidateIdx < prefixIdx; - const BYTE* const matchPtr = (extDict ? dictStart - dictIdx : prefixPtr - prefixIdx) + matchCandidateIdx; + const BYTE* const matchPtr = extDict ? dictStart + (matchCandidateIdx - dictIdx) : prefixPtr + (matchCandidateIdx - prefixIdx); if (LZ4_read32(matchPtr) == pattern) { /* good candidate */ const BYTE* const iLimit = extDict ? dictEnd : iHighLimit; size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern); @@ -399,8 +412,9 @@ LZ4HC_InsertAndGetWiderMatch ( if ((size_t)(ip - prefixPtr) + prefixIdx - matchIndex > LZ4_DISTANCE_MAX) break; assert(maxML < 2 GB); longest = (int)maxML; - *matchpos = prefixPtr - prefixIdx + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ + offset = (int)(ipIndex - matchIndex); *startpos = ip; + DEBUGLOG(7, "Found repeat pattern match of len=%i, offset=%i", longest, offset); } { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex); if (distToNextPattern > matchIndex) break; /* avoid overflow */ @@ -422,6 +436,7 @@ LZ4HC_InsertAndGetWiderMatch ( U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; assert(dictEndOffset <= 1 GB); matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset; + if (dictMatchIndex>0) DEBUGLOG(7, "dictEndOffset = %zu, dictMatchIndex = %u => relative matchIndex = %i", dictEndOffset, dictMatchIndex, (int)dictMatchIndex - (int)dictEndOffset); while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) { const BYTE* const matchPtr = dictCtx->prefixStart - dictCtx->dictLimit + dictMatchIndex; @@ -435,8 +450,9 @@ LZ4HC_InsertAndGetWiderMatch ( mlt -= back; if (mlt > longest) { longest = mlt; - *matchpos = prefixPtr - prefixIdx + matchIndex + back; + offset = (int)(ipIndex - matchIndex); *startpos = ip + back; + DEBUGLOG(7, "found match of length %i within extDictCtx", longest); } } { U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex); @@ -444,22 +460,27 @@ LZ4HC_InsertAndGetWiderMatch ( matchIndex -= nextOffset; } } } - return longest; + { LZ4HC_match_t md; + assert(longest >= 0); + md.len = longest; + md.off = offset; + return md; + } } -LZ4_FORCE_INLINE int +LZ4_FORCE_INLINE LZ4HC_match_t LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */ const BYTE* const ip, const BYTE* const iLimit, - const BYTE** matchpos, const int maxNbAttempts, const int patternAnalysis, const dictCtx_directive dict) { const BYTE* uselessPtr = ip; + DEBUGLOG(7, "LZ4HC_InsertAndFindBestMatch"); /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), * but this won't be the case here, as we define iLowLimit==ip, * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ - return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, 0 /*chainSwap*/, dict, favorCompressionRatio); + return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, &uselessPtr, maxNbAttempts, patternAnalysis, 0 /*chainSwap*/, dict, favorCompressionRatio); } /* LZ4HC_encodeSequence() : @@ -470,7 +491,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( BYTE** _op, const BYTE** _anchor, int matchLength, - const BYTE* const match, + int offset, limitedOutput_directive limit, BYTE* oend) { @@ -491,9 +512,9 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( U32 const cost = 1 + llAdd + ll + 2 + mlAdd; if (start==NULL) start = anchor; /* only works for single segment */ /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */ - DEBUGLOG(6, "pos:%7u -- literals:%4u, match:%4i, offset:%5u, cost:%4u + %5u", + DEBUGLOG(6, "pos:%7u -- literals:%4u, match:%4i, offset:%5i, cost:%4u + %5u", pos, - (U32)(ip - anchor), matchLength, (U32)(ip-match), + (U32)(ip - anchor), matchLength, offset, cost, totalCost); totalCost += cost; #endif @@ -521,8 +542,9 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( op += length; /* Encode Offset */ - assert( (ip - match) <= LZ4_DISTANCE_MAX ); /* note : consider providing offset as a value, rather than as a pointer difference */ - LZ4_writeLE16(op, (U16)(ip - match)); op += 2; + assert(offset <= LZ4_DISTANCE_MAX ); + assert(offset > 0); + LZ4_writeLE16(op, (U16)(offset)); op += 2; /* Encode MatchLength */ assert(matchLength >= MINMATCH); @@ -575,127 +597,116 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( BYTE* op = (BYTE*) dest; BYTE* oend = op + maxOutputSize; - int ml0, ml, ml2, ml3; const BYTE* start0; - const BYTE* ref0; - const BYTE* ref = NULL; const BYTE* start2 = NULL; - const BYTE* ref2 = NULL; const BYTE* start3 = NULL; - const BYTE* ref3 = NULL; + LZ4HC_match_t m0, m1, m2, m3; + const LZ4HC_match_t nomatch = {0, 0}; /* init */ + DEBUGLOG(5, "LZ4HC_compress_hashChain (dict?=>%i)", dict); *srcSizePtr = 0; if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ /* Main Loop */ while (ip <= mflimit) { - ml = LZ4HC_InsertAndFindBestMatch(ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis, dict); - if (ml encode ML1 */ + if (m2.len <= m1.len) { /* No better match => encode ML1 immediately */ optr = op; - if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), m1.len, m1.off, limit, oend)) goto _dest_overflow; continue; } if (start0 < ip) { /* first match was skipped at least once */ - if (start2 < ip + ml0) { /* squeezing ML1 between ML0(original ML1) and ML2 */ - ip = start0; ref = ref0; ml = ml0; /* restore initial ML1 */ + if (start2 < ip + m0.len) { /* squeezing ML1 between ML0(original ML1) and ML2 */ + ip = start0; m1 = m0; /* restore initial Match1 */ } } /* Here, start0==ip */ if ((start2 - ip) < 3) { /* First Match too small : removed */ - ml = ml2; ip = start2; - ref =ref2; + m1 = m2; goto _Search2; } _Search3: - /* At this stage, we have : - * ml2 > ml1, and - * ip1+3 <= ip2 (usually < ip1+ml1) */ if ((start2 - ip) < OPTIMAL_ML) { int correction; - int new_ml = ml; + int new_ml = m1.len; if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; - if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; + if (ip+new_ml > start2 + m2.len - MINMATCH) + new_ml = (int)(start2 - ip) + m2.len - MINMATCH; correction = new_ml - (int)(start2 - ip); if (correction > 0) { start2 += correction; - ref2 += correction; - ml2 -= correction; + m2.len -= correction; } } - /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ - if (start2 + ml2 <= mflimit) { - ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, - start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, + if (start2 + m2.len <= mflimit) { + m3 = LZ4HC_InsertAndGetWiderMatch(ctx, + start2 + m2.len - 3, start2, matchlimit, m2.len, &start3, maxNbAttempts, patternAnalysis, 0, dict, favorCompressionRatio); } else { - ml3 = ml2; + m3 = nomatch; /* do not search further */ } - if (ml3 == ml2) { /* No better match => encode ML1 and ML2 */ + if (m3.len <= m2.len) { /* No better match => encode ML1 and ML2 */ /* ip & ref are known; Now for ml */ - if (start2 < ip+ml) ml = (int)(start2 - ip); + if (start2 < ip+m1.len) m1.len = (int)(start2 - ip); /* Now, encode 2 sequences */ optr = op; - if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), m1.len, m1.off, limit, oend)) + goto _dest_overflow; ip = start2; optr = op; - if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml2, ref2, limit, oend)) { - ml = ml2; - ref = ref2; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), m2.len, m2.off, limit, oend)) { + m1 = m2; goto _dest_overflow; } continue; } - if (start3 < ip+ml+3) { /* Not enough space for match 2 : remove it */ - if (start3 >= (ip+ml)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ - if (start2 < ip+ml) { - int correction = (int)(ip+ml - start2); + if (start3 < ip+m1.len+3) { /* Not enough space for match 2 : remove it */ + if (start3 >= (ip+m1.len)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ + if (start2 < ip+m1.len) { + int correction = (int)(ip+m1.len - start2); start2 += correction; - ref2 += correction; - ml2 -= correction; - if (ml2 < MINMATCH) { + m2.len -= correction; + if (m2.len < MINMATCH) { start2 = start3; - ref2 = ref3; - ml2 = ml3; + m2 = m3; } } optr = op; - if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), m1.len, m1.off, limit, oend)) goto _dest_overflow; ip = start3; - ref = ref3; - ml = ml3; + m1 = m3; start0 = start2; - ref0 = ref2; - ml0 = ml2; + m0 = m2; goto _Search2; } start2 = start3; - ref2 = ref3; - ml2 = ml3; + m2 = m3; goto _Search3; } @@ -704,29 +715,29 @@ _Search3: * let's write the first one ML1. * ip & ref are known; Now decide ml. */ - if (start2 < ip+ml) { + if (start2 < ip+m1.len) { if ((start2 - ip) < OPTIMAL_ML) { int correction; - if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; - if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH; - correction = ml - (int)(start2 - ip); + if (m1.len > OPTIMAL_ML) m1.len = OPTIMAL_ML; + if (ip + m1.len > start2 + m2.len - MINMATCH) + m1.len = (int)(start2 - ip) + m2.len - MINMATCH; + correction = m1.len - (int)(start2 - ip); if (correction > 0) { start2 += correction; - ref2 += correction; - ml2 -= correction; + m2.len -= correction; } } else { - ml = (int)(start2 - ip); + m1.len = (int)(start2 - ip); } } optr = op; - if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), m1.len, m1.off, limit, oend)) goto _dest_overflow; /* ML2 becomes ML1 */ - ip = start2; ref = ref2; ml = ml2; + ip = start2; m1 = m2; /* ML3 becomes ML2 */ - start2 = start3; ref2 = ref3; ml2 = ml3; + start2 = start3; m2 = m3; /* let's find a new ML3 */ goto _Search3; @@ -777,10 +788,10 @@ _dest_overflow: /* ll validated; now adjust match length */ size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); - assert(maxMlSize < INT_MAX); assert(ml >= 0); - if ((size_t)ml > maxMlSize) ml = (int)maxMlSize; - if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + ml >= MFLIMIT) { - LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, notLimited, oend); + assert(maxMlSize < INT_MAX); assert(m1.len >= 0); + if ((size_t)m1.len > maxMlSize) m1.len = (int)maxMlSize; + if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + m1.len >= MFLIMIT) { + LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), m1.len, m1.off, notLimited, oend); } } goto _last_literals; } @@ -798,16 +809,17 @@ static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx, const HCfavor_e favorDecSpeed); -LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( - LZ4HC_CCtx_internal* const ctx, - const char* const src, - char* const dst, - int* const srcSizePtr, - int const dstCapacity, - int cLevel, - const limitedOutput_directive limit, - const dictCtx_directive dict - ) +LZ4_FORCE_INLINE int +LZ4HC_compress_generic_internal ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + const limitedOutput_directive limit, + const dictCtx_directive dict + ) { typedef enum { lz4hc, lz4opt } lz4hc_strat_e; typedef struct { @@ -831,8 +843,8 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( { lz4opt,16384,LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ }; - DEBUGLOG(4, "LZ4HC_compress_generic(ctx=%p, src=%p, srcSize=%d, limit=%d)", - ctx, src, *srcSizePtr, limit); + DEBUGLOG(5, "LZ4HC_compress_generic_internal(src=%p, srcSize=%d)", + src, *srcSizePtr); if (limit == fillOutput && dstCapacity < 1) return 0; /* Impossible to store anything */ if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */ @@ -966,6 +978,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in LZ4_streamHC_t state; LZ4_streamHC_t* const statePtr = &state; #endif + DEBUGLOG(5, "LZ4_compress_HC") cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 FREEMEM(statePtr); @@ -1034,7 +1047,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { LZ4HC_CCtx_internal* const s = &LZ4_streamHCPtr->internal_donotuse; - DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel); + DEBUGLOG(5, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel); if (s->dirty) { LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); } else { @@ -1150,6 +1163,7 @@ LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int srcSize, int dstCapacity) { + DEBUGLOG(5, "LZ4_compress_HC_continue"); if (dstCapacity < LZ4_compressBound(srcSize)) return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, limitedOutput); else @@ -1299,10 +1313,6 @@ LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen) } -typedef struct { - int off; - int len; -} LZ4HC_match_t; LZ4_FORCE_INLINE LZ4HC_match_t LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx, @@ -1311,19 +1321,16 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx, const dictCtx_directive dict, const HCfavor_e favorDecSpeed) { - LZ4HC_match_t match = { 0 , 0 }; - const BYTE* matchPtr = NULL; + LZ4HC_match_t const match0 = { 0 , 0 }; /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), * but this won't be the case here, as we define iLowLimit==ip, - * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ - int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /*patternAnalysis*/, 1 /*chainSwap*/, dict, favorDecSpeed); - if (matchLength <= minLen) return match; + ** so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ + LZ4HC_match_t md = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &ip, nbSearches, 1 /*patternAnalysis*/, 1 /*chainSwap*/, dict, favorDecSpeed); + if (md.len <= minLen) return match0; if (favorDecSpeed) { - if ((matchLength>18) & (matchLength<=36)) matchLength=18; /* favor shortcut */ + if ((md.len>18) & (md.len<=36)) md.len=18; /* favor shortcut */ } - match.len = matchLength; - match.off = (int)(ip-matchPtr); - return match; + return md; } @@ -1356,7 +1363,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, BYTE* opSaved = (BYTE*) dst; BYTE* oend = op + dstCapacity; int ovml = MINMATCH; /* overflow - last sequence */ - const BYTE* ovref = NULL; + int ovoff = 0; /* init */ #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 @@ -1379,11 +1386,10 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, if ((size_t)firstMatch.len > sufficient_len) { /* good enough solution : immediate encoding */ int const firstML = firstMatch.len; - const BYTE* const matchPos = ip - firstMatch.off; opSaved = op; - if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, matchPos, limit, oend) ) { /* updates ip, op and anchor */ + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, firstMatch.off, limit, oend) ) { /* updates ip, op and anchor */ ovml = firstML; - ovref = matchPos; + ovoff = firstMatch.off; goto _dest_overflow; } continue; @@ -1557,9 +1563,9 @@ encode: /* cur, last_match_pos, best_mlen, best_off must be set */ assert(ml >= MINMATCH); assert((offset >= 1) && (offset <= LZ4_DISTANCE_MAX)); opSaved = op; - if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ip - offset, limit, oend) ) { /* updates ip, op and anchor */ + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, offset, limit, oend) ) { /* updates ip, op and anchor */ ovml = ml; - ovref = ip - offset; + ovoff = offset; goto _dest_overflow; } } } } /* while (ip <= mflimit) */ @@ -1618,7 +1624,7 @@ if (limit == fillOutput) { if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + ovml >= MFLIMIT) { DEBUGLOG(6, "Space to end : %i + ml (%i)", (int)((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1), ovml); DEBUGLOG(6, "Before : ip = %p, anchor = %p", ip, anchor); - LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ovml, ovref, notLimited, oend); + LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ovml, ovoff, notLimited, oend); DEBUGLOG(6, "After : ip = %p, anchor = %p", ip, anchor); } } goto _last_literals; diff --git a/CompressSave/README.md b/CompressSave/README.md index 281bf63..1ce8192 100644 --- a/CompressSave/README.md +++ b/CompressSave/README.md @@ -7,6 +7,13 @@ ## Updates +### 1.3.0 +* Separate config entries for manual save and auto save. +* Now you can still get speed benefit while setting compression type to `None` for auto saves, and for manual saves if using the new `Save` button. + * Adds a `nonewrap.dll` for this function. +* Update `LZ4` and `Zstd` library to latest version. +* `lz4wrap.dll` and `zstdwrap.dll` are compiled using `-O3` instead of `-Os`, expect to be slightly faster but larger. + ### 1.2.2 * Fix #4, a bug caused by non-ASCII UTF-8 characters. * Remove use of Harmony.UnpatchAll() to avoid warnings in BepInEx log. @@ -111,8 +118,9 @@ ## Usage -* All autosaves are compressed +* You can set compression type for manual saves and auto saves individually. * Manual saves are compressed while using the new `Save` button. +* You can still get speed benefit while setting compression type to `None` for auto saves, and for manual saves if using the new `Save` button. * You can decompress saves on load panel. * Remember to backup your save(use original save button) before updating game to avoid loading failure. @@ -128,7 +136,8 @@ ## 使用说明 -* 所有自动存档都会被压缩。 +* 手动和自动存档都可以分开设置压缩方式。 * 手动存档使用新加的保存按钮即可压缩保存。 +* 即使设置为不压缩，自动存档、以及使用新加的保存按钮手动保存也可以获得速度提升。 * 可以在读取存档面板解压存档。 * 如果游戏有版本更新记得先备份存档(使用原保存按钮)以免更新后无法读取存档。 diff --git a/CompressSave/Wrapper/CompressionStream.cs b/CompressSave/Wrapper/CompressionStream.cs index f18afb6..c95bcd9 100644 --- a/CompressSave/Wrapper/CompressionStream.cs +++ b/CompressSave/Wrapper/CompressionStream.cs @@ -56,15 +56,15 @@ public class CompressionStream : Stream public byte[] outBuffer; } - public static CompressBuffer CreateBuffer(WrapperDefines wrapper, int ExBufferSize = 4 * MB) + public static CompressBuffer CreateBuffer(int outBufferSize, int exBufferSize = 4 * MB) { try { return new CompressBuffer { - outBuffer = new byte[wrapper.CompressBufferBound(ExBufferSize)], - readBuffer = new byte[ExBufferSize], - writeBuffer = new byte[ExBufferSize], + outBuffer = new byte[outBufferSize], + readBuffer = new byte[exBufferSize], + writeBuffer = new byte[exBufferSize], }; } catch (Exception e) @@ -93,10 +93,10 @@ public class CompressionStream : Stream } } - void InitBuffer(byte[] readBuffer, byte[] writeBuffer, byte[] outBuffer) + void InitBuffer(byte[] readBuffer, byte[] writeBuffer, byte[] outputBuffer) { doubleBuffer = new DoubleBuffer(readBuffer ?? new byte[4 * MB], writeBuffer ?? new byte[4 * MB], Compress); - this.outBuffer = outBuffer ?? new byte[wrapper.CompressBufferBound(writeBuffer.Length)]; + outBuffer = outputBuffer ?? new byte[wrapper.CompressBufferBound(writeBuffer?.Length ?? 4 * MB)]; BufferWriter = new BufferWriter(doubleBuffer,this); } diff --git a/CompressSave/package/manifest.json b/CompressSave/package/manifest.json index a6ff76a..af36ef8 100644 --- a/CompressSave/package/manifest.json +++ b/CompressSave/package/manifest.json @@ -1,6 +1,6 @@ { "name": "CompressSave", - "version_number": "1.2.2", + "version_number": "1.3.0", "website_url": "https://github.com/soarqin/DSP_Mods/tree/master/CompressSave", "description": "Compress game saves to reduce space use and boost save speed / 压缩游戏存档以降低空间使用并提升保存速度", "dependencies": ["xiaoye97-BepInEx-5.4.17"] diff --git a/CompressSave/package/plugins/x64/lz4wrap.dll b/CompressSave/package/plugins/x64/lz4wrap.dll index 8d7d800..28b26b9 100644 Binary files a/CompressSave/package/plugins/x64/lz4wrap.dll and b/CompressSave/package/plugins/x64/lz4wrap.dll differ diff --git a/CompressSave/package/plugins/x64/zstdwrap.dll b/CompressSave/package/plugins/x64/zstdwrap.dll index a0ff270..2f1e2fc 100644 Binary files a/CompressSave/package/plugins/x64/zstdwrap.dll and b/CompressSave/package/plugins/x64/zstdwrap.dll differ