mirror of
https://github.com/soarqin/DSP_Mods.git
synced 2025-12-09 04:13:32 +08:00
WIP: CompressSave 1.2.0
This commit is contained in:
50
CompressSave/Wrapper/BlackHoleStream.cs
Normal file
50
CompressSave/Wrapper/BlackHoleStream.cs
Normal file
@@ -0,0 +1,50 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace CompressSave.Wrapper;
|
||||
|
||||
class BlackHoleStream : Stream
|
||||
{
|
||||
private long length;
|
||||
|
||||
public override bool CanRead => true;
|
||||
|
||||
public override bool CanSeek => false;
|
||||
|
||||
public override bool CanWrite => true;
|
||||
|
||||
public override long Length => length;
|
||||
|
||||
public override long Position { get; set; }
|
||||
|
||||
public BlackHoleStream()
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public override void Flush()
|
||||
{
|
||||
;
|
||||
}
|
||||
|
||||
public override int Read(byte[] buffer, int offset, int count)
|
||||
{
|
||||
return count;
|
||||
}
|
||||
|
||||
public override long Seek(long offset, SeekOrigin origin)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override void SetLength(long value)
|
||||
{
|
||||
length = value;
|
||||
}
|
||||
public byte[] testBuffer = new byte[1024 * 1024];
|
||||
|
||||
public override void Write(byte[] buffer, int offset, int count)
|
||||
{
|
||||
Array.Copy(buffer, offset, testBuffer, 0, Math.Min(count, testBuffer.Length));
|
||||
}
|
||||
}
|
||||
326
CompressSave/Wrapper/BufferWriter.cs
Normal file
326
CompressSave/Wrapper/BufferWriter.cs
Normal file
@@ -0,0 +1,326 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace CompressSave.Wrapper;
|
||||
|
||||
public unsafe class BufferWriter : BinaryWriter
|
||||
{
|
||||
ByteSpan currentBuffer => doubleBuffer.writeBuffer;
|
||||
|
||||
DoubleBuffer doubleBuffer;
|
||||
|
||||
private Encoding _encoding;
|
||||
|
||||
private Encoder encoder;
|
||||
|
||||
byte[] Buffer => currentBuffer.Buffer;
|
||||
|
||||
long SuplusCapacity => endPos - curPos;
|
||||
|
||||
long swapedBytes = 0;
|
||||
|
||||
public long WriteSum => swapedBytes + curPos - startPos;
|
||||
|
||||
public override Stream BaseStream => _baseStream;
|
||||
|
||||
public override void Write(char[] chars, int index, int count)
|
||||
{
|
||||
if (chars == null)
|
||||
{
|
||||
throw new ArgumentNullException("chars");
|
||||
}
|
||||
byte[] bytes = _encoding.GetBytes(chars, index, count);
|
||||
Write(bytes);
|
||||
}
|
||||
|
||||
byte* curPos;
|
||||
byte* endPos;
|
||||
byte* startPos;
|
||||
private Stream _baseStream;
|
||||
|
||||
public BufferWriter(DoubleBuffer doubleBuffer, CompressionStream outStream)
|
||||
: this(doubleBuffer, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true), outStream)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
BufferWriter(DoubleBuffer buffer , UTF8Encoding encoding, CompressionStream outStream) : base(Stream.Null, encoding)
|
||||
{
|
||||
_baseStream = outStream;
|
||||
swapedBytes = 0;
|
||||
doubleBuffer = buffer;
|
||||
RefreshStatus();
|
||||
_encoding = encoding;
|
||||
encoder = _encoding.GetEncoder();
|
||||
}
|
||||
|
||||
void SwapBuffer()
|
||||
{
|
||||
currentBuffer.Position = 0;
|
||||
|
||||
currentBuffer.Length = (int)(curPos - startPos);
|
||||
swapedBytes += currentBuffer.Length;
|
||||
doubleBuffer.SwapBuffer();
|
||||
RefreshStatus();
|
||||
}
|
||||
|
||||
void RefreshStatus()
|
||||
{
|
||||
startPos = (byte*)Unsafe.AsPointer(ref Buffer[0]);
|
||||
curPos = startPos;
|
||||
endPos = (byte*)Unsafe.AsPointer(ref Buffer[Buffer.Length - 1]) + 1;
|
||||
}
|
||||
|
||||
void CheckCapacityAndSwap(int requiredCapacity)
|
||||
{
|
||||
if (SuplusCapacity < requiredCapacity)
|
||||
{
|
||||
SwapBuffer();
|
||||
}
|
||||
}
|
||||
|
||||
public override void Write(byte value)
|
||||
{
|
||||
CheckCapacityAndSwap(1);
|
||||
*(curPos++) = value;
|
||||
}
|
||||
|
||||
public override void Write(bool value) => Write((byte)(value ? 1 : 0));
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
if (disposing)
|
||||
{
|
||||
SwapBuffer();
|
||||
}
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
|
||||
public override void Close()
|
||||
{
|
||||
Dispose(disposing: true);
|
||||
}
|
||||
|
||||
public override void Flush()
|
||||
{
|
||||
SwapBuffer();
|
||||
}
|
||||
|
||||
public override long Seek(int offset, SeekOrigin origin)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override void Write(sbyte value) => Write((byte)value);
|
||||
|
||||
public override void Write(byte[] _buffer) => Write(_buffer, 0, _buffer.Length);
|
||||
|
||||
|
||||
public override void Write(byte[] _buffer, int index, int count)
|
||||
{
|
||||
if (_buffer == null)
|
||||
{
|
||||
throw new ArgumentNullException("buffer");
|
||||
}
|
||||
fixed (byte* start = _buffer)
|
||||
{
|
||||
byte* srcPos = start + index;
|
||||
while (SuplusCapacity <= count)
|
||||
{
|
||||
int dstSuplus = (int)SuplusCapacity;
|
||||
//Array.Copy(_buffer, index + writed, Buffer, Position, SuplusCapacity);
|
||||
Unsafe.CopyBlock(curPos, srcPos, (uint)dstSuplus);
|
||||
count -= dstSuplus;
|
||||
srcPos += dstSuplus;
|
||||
curPos = endPos;
|
||||
SwapBuffer();
|
||||
}
|
||||
Unsafe.CopyBlock(curPos, srcPos, (uint)count);
|
||||
curPos += count;
|
||||
}
|
||||
}
|
||||
|
||||
public unsafe override void Write(char ch)
|
||||
{
|
||||
if (char.IsSurrogate(ch))
|
||||
{
|
||||
throw new ArgumentException("Arg_SurrogatesNotAllowedAsSingleChar");
|
||||
}
|
||||
|
||||
CheckCapacityAndSwap(4);
|
||||
|
||||
curPos += encoder.GetBytes(&ch, 1, curPos, (int)SuplusCapacity, flush: true);
|
||||
}
|
||||
|
||||
//slow
|
||||
public override void Write(char[] chars)
|
||||
{
|
||||
if (chars == null)
|
||||
{
|
||||
throw new ArgumentNullException("chars");
|
||||
}
|
||||
byte[] bytes = _encoding.GetBytes(chars, 0, chars.Length);
|
||||
Write(bytes);
|
||||
}
|
||||
|
||||
public unsafe override void Write(double value)
|
||||
{
|
||||
CheckCapacityAndSwap(8);
|
||||
ulong num = (ulong)(*(long*)(&value));
|
||||
*(curPos++) = (byte)num;
|
||||
*(curPos++) = (byte)(num >> 8);
|
||||
*(curPos++) = (byte)(num >> 16);
|
||||
*(curPos++) = (byte)(num >> 24);
|
||||
*(curPos++) = (byte)(num >> 32);
|
||||
*(curPos++) = (byte)(num >> 40);
|
||||
*(curPos++) = (byte)(num >> 48);
|
||||
*(curPos++) = (byte)(num >> 56);
|
||||
}
|
||||
|
||||
//slow
|
||||
public override void Write(decimal d)
|
||||
{
|
||||
CheckCapacityAndSwap(16);
|
||||
int[] bits = decimal.GetBits(d);
|
||||
|
||||
Write(bits[0]);
|
||||
Write(bits[1]);
|
||||
Write(bits[2]);
|
||||
Write(bits[3]);
|
||||
}
|
||||
|
||||
|
||||
public override void Write(short value)
|
||||
{
|
||||
CheckCapacityAndSwap(2);
|
||||
*(curPos++) = (byte)value;
|
||||
*(curPos++) = (byte)(value >> 8);
|
||||
}
|
||||
|
||||
public override void Write(ushort value)
|
||||
{
|
||||
CheckCapacityAndSwap(2);
|
||||
*(curPos++) = (byte)value;
|
||||
*(curPos++) = (byte)(value >> 8);
|
||||
}
|
||||
|
||||
|
||||
public override void Write(int value)
|
||||
{
|
||||
if (SuplusCapacity < 4)
|
||||
{
|
||||
SwapBuffer();
|
||||
}
|
||||
*(curPos++) = (byte)value;
|
||||
*(curPos++) = (byte)(value >> 8);
|
||||
*(curPos++) = (byte)(value >> 16);
|
||||
*(curPos++) = (byte)(value >> 24);
|
||||
}
|
||||
|
||||
public override void Write(uint value)
|
||||
{
|
||||
CheckCapacityAndSwap(4);
|
||||
*(curPos++) = (byte)value;
|
||||
*(curPos++) = (byte)(value >> 8);
|
||||
*(curPos++) = (byte)(value >> 16);
|
||||
*(curPos++) = (byte)(value >> 24);
|
||||
}
|
||||
|
||||
|
||||
public override void Write(long value)
|
||||
{
|
||||
CheckCapacityAndSwap(8);
|
||||
*(curPos++) = (byte)value;
|
||||
*(curPos++) = (byte)(value >> 8);
|
||||
*(curPos++) = (byte)(value >> 16);
|
||||
*(curPos++) = (byte)(value >> 24);
|
||||
*(curPos++) = (byte)(value >> 32);
|
||||
*(curPos++) = (byte)(value >> 40);
|
||||
*(curPos++) = (byte)(value >> 48);
|
||||
*(curPos++) = (byte)(value >> 56);
|
||||
}
|
||||
|
||||
public override void Write(ulong value)
|
||||
{
|
||||
CheckCapacityAndSwap(8);
|
||||
*(curPos++) = (byte)value;
|
||||
*(curPos++) = (byte)(value >> 8);
|
||||
*(curPos++) = (byte)(value >> 16);
|
||||
*(curPos++) = (byte)(value >> 24);
|
||||
*(curPos++) = (byte)(value >> 32);
|
||||
*(curPos++) = (byte)(value >> 40);
|
||||
*(curPos++) = (byte)(value >> 48);
|
||||
*(curPos++) = (byte)(value >> 56);
|
||||
}
|
||||
|
||||
public unsafe override void Write(float value)
|
||||
{
|
||||
if (SuplusCapacity < 4)
|
||||
{
|
||||
SwapBuffer();
|
||||
}
|
||||
uint num = *(uint*)(&value);
|
||||
*(curPos++) = (byte)num;
|
||||
*(curPos++) = (byte)(num >> 8);
|
||||
*(curPos++) = (byte)(num >> 16);
|
||||
*(curPos++) = (byte)(num >> 24);
|
||||
}
|
||||
|
||||
|
||||
//slow
|
||||
public unsafe override void Write(string value)
|
||||
{
|
||||
if (value == null)
|
||||
{
|
||||
throw new ArgumentNullException("value");
|
||||
}
|
||||
int byteCount = _encoding.GetByteCount(value);
|
||||
Write7BitEncodedInt(byteCount);
|
||||
{
|
||||
var dstSuplus = (int)SuplusCapacity;
|
||||
if (byteCount <= dstSuplus)
|
||||
{
|
||||
fixed (char* start = value)
|
||||
{
|
||||
int Wcount = _encoding.GetBytes(start, value.Length, curPos, dstSuplus);
|
||||
curPos += Wcount;
|
||||
//Console.WriteLine($"Using quick write!");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int charIndex = 0;
|
||||
bool completed;
|
||||
fixed (char* chars = value)
|
||||
{
|
||||
do
|
||||
{
|
||||
encoder.Convert(chars + charIndex, value.Length - charIndex,
|
||||
curPos, (int)SuplusCapacity, true,
|
||||
out int charsConsumed, out int bytesWritten, out completed);
|
||||
charIndex += charsConsumed;
|
||||
curPos += bytesWritten;
|
||||
//Console.WriteLine($"charsConsumed{charsConsumed} charIndex{charIndex} bytesWritten{bytesWritten} position{position} suplusCapacity{suplusCapacity}");
|
||||
|
||||
if (SuplusCapacity <= 0)
|
||||
SwapBuffer();
|
||||
} while (!completed);
|
||||
}
|
||||
encoder.Reset(); //flush
|
||||
}
|
||||
|
||||
|
||||
|
||||
protected new void Write7BitEncodedInt(int value)
|
||||
{
|
||||
uint num;
|
||||
for (num = (uint)value; num >= 128; num >>= 7)
|
||||
{
|
||||
Write((byte)(num | 0x80));
|
||||
}
|
||||
Write((byte)num);
|
||||
}
|
||||
}
|
||||
117
CompressSave/Wrapper/BufferedStream.cs
Normal file
117
CompressSave/Wrapper/BufferedStream.cs
Normal file
@@ -0,0 +1,117 @@
|
||||
namespace CompressSave.Wrapper;
|
||||
|
||||
//public class BufferedFileStream : FileStream
|
||||
//{
|
||||
// public override bool CanTimeout => base.CanTimeout;
|
||||
|
||||
// public override int ReadTimeout { get => base.ReadTimeout; set => base.ReadTimeout = value; }
|
||||
// public override int WriteTimeout { get => base.WriteTimeout; set => base.WriteTimeout = value; }
|
||||
|
||||
// public override long Position { get => base.Position; set => base.Position = value; }
|
||||
|
||||
// public override SafeFileHandle SafeFileHandle => base.SafeFileHandle;
|
||||
|
||||
// public override IAsyncResult BeginRead(byte[] array, int offset, int numBytes, AsyncCallback userCallback, object stateObject)
|
||||
// {
|
||||
// return base.BeginRead(array, offset, numBytes, userCallback, stateObject);
|
||||
// }
|
||||
|
||||
// public override IAsyncResult BeginWrite(byte[] array, int offset, int numBytes, AsyncCallback userCallback, object stateObject)
|
||||
// {
|
||||
// return base.BeginWrite(array, offset, numBytes, userCallback, stateObject);
|
||||
// }
|
||||
|
||||
// public override void Close()
|
||||
// {
|
||||
// var bs = new BufferedStream(this);
|
||||
|
||||
|
||||
// base.Close();
|
||||
// }
|
||||
|
||||
// public override bool Equals(object obj)
|
||||
// {
|
||||
// return base.Equals(obj);
|
||||
// }
|
||||
|
||||
// public override void Flush()
|
||||
// {
|
||||
// base.Flush();
|
||||
// }
|
||||
|
||||
// public override void Flush(bool flushToDisk)
|
||||
// {
|
||||
// base.Flush(flushToDisk);
|
||||
// }
|
||||
|
||||
// public override Task FlushAsync(CancellationToken cancellationToken)
|
||||
// {
|
||||
// return base.FlushAsync(cancellationToken);
|
||||
// }
|
||||
|
||||
// public override int GetHashCode()
|
||||
// {
|
||||
// return base.GetHashCode();
|
||||
// }
|
||||
|
||||
// public override object InitializeLifetimeService()
|
||||
// {
|
||||
// return base.InitializeLifetimeService();
|
||||
// }
|
||||
|
||||
// public override void Lock(long position, long length)
|
||||
// {
|
||||
// base.Lock(position, length);
|
||||
// }
|
||||
|
||||
// public override int Read(byte[] array, int offset, int count)
|
||||
// {
|
||||
// return base.Read(array, offset, count);
|
||||
// }
|
||||
|
||||
// public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
|
||||
// {
|
||||
// return base.ReadAsync(buffer, offset, count, cancellationToken);
|
||||
// }
|
||||
|
||||
// public override long Seek(long offset, SeekOrigin origin)
|
||||
// {
|
||||
// return base.Seek(offset, origin);
|
||||
// }
|
||||
|
||||
// public override void SetLength(long value)
|
||||
// {
|
||||
// base.SetLength(value);
|
||||
// }
|
||||
|
||||
// public override string ToString()
|
||||
// {
|
||||
// return base.ToString();
|
||||
// }
|
||||
|
||||
// public override void Unlock(long position, long length)
|
||||
// {
|
||||
// base.Unlock(position, length);
|
||||
// }
|
||||
|
||||
// public override void Write(byte[] array, int offset, int count)
|
||||
// {
|
||||
// base.Write(array, offset, count);
|
||||
// }
|
||||
|
||||
// public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
|
||||
// {
|
||||
// return base.WriteAsync(buffer, offset, count, cancellationToken);
|
||||
// }
|
||||
|
||||
// public override void WriteByte(byte value)
|
||||
// {
|
||||
// base.WriteByte(value);
|
||||
// }
|
||||
|
||||
// protected override void Dispose(bool disposing)
|
||||
// {
|
||||
// base.Dispose(disposing);
|
||||
// }
|
||||
|
||||
//}
|
||||
225
CompressSave/Wrapper/CompressionStream.cs
Normal file
225
CompressSave/Wrapper/CompressionStream.cs
Normal file
@@ -0,0 +1,225 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
|
||||
namespace CompressSave.Wrapper;
|
||||
|
||||
public class CompressionStream : Stream
|
||||
{
|
||||
public WrapperDefines wrapper;
|
||||
|
||||
public const int MB = 1024 * 1024;
|
||||
public override bool CanRead => false;
|
||||
|
||||
public override bool CanSeek => false;
|
||||
|
||||
public override bool CanWrite => true;
|
||||
|
||||
public override long Length => totalWrite;
|
||||
|
||||
// only use for game statistics
|
||||
public override long Position { get => BufferWriter.WriteSum; set => new NotImplementedException(); }
|
||||
|
||||
readonly Stream outStream;
|
||||
|
||||
long totalWrite = 0;
|
||||
bool useMultiThread;
|
||||
DoubleBuffer doubleBuffer;
|
||||
|
||||
private byte[] outBuffer;
|
||||
|
||||
IntPtr cctx;
|
||||
long lastError = 0;
|
||||
bool stopWorker = true;
|
||||
Thread compressThread;
|
||||
|
||||
public bool HasError()
|
||||
{
|
||||
return lastError != 0;
|
||||
}
|
||||
|
||||
public void HandleError(long errorCode)
|
||||
{
|
||||
if (errorCode < 0)
|
||||
{
|
||||
wrapper.CompressContextFree(cctx);
|
||||
cctx = IntPtr.Zero;
|
||||
lastError = errorCode;
|
||||
throw new Exception(errorCode.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
public struct CompressBuffer
|
||||
{
|
||||
public byte[] readBuffer;
|
||||
public byte[] writeBuffer;
|
||||
public byte[] outBuffer;
|
||||
}
|
||||
|
||||
public static CompressBuffer CreateBuffer(WrapperDefines wrapper, int ExBufferSize = 4 * MB)
|
||||
{
|
||||
try
|
||||
{
|
||||
return new CompressBuffer
|
||||
{
|
||||
outBuffer = new byte[wrapper.CompressBufferBound(ExBufferSize) + 1],
|
||||
readBuffer = new byte[ExBufferSize],
|
||||
writeBuffer = new byte[ExBufferSize],
|
||||
};
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine(e.ToString());
|
||||
}
|
||||
return new CompressBuffer();
|
||||
}
|
||||
|
||||
public BufferWriter BufferWriter => bfferWriter;
|
||||
BufferWriter bfferWriter;
|
||||
|
||||
public CompressionStream(WrapperDefines wrap, int compressionLevel, Stream outStream, CompressBuffer compressBuffer, bool useMultiThread)
|
||||
{
|
||||
this.wrapper = wrap;
|
||||
this.outStream = outStream;
|
||||
InitBuffer(compressBuffer.readBuffer, compressBuffer.writeBuffer, compressBuffer.outBuffer);
|
||||
long writeSize = wrapper.CompressBegin(out cctx, compressionLevel, outBuffer, outBuffer.Length);
|
||||
HandleError(writeSize);
|
||||
outStream.Write(outBuffer, 0, (int)writeSize);
|
||||
this.useMultiThread = useMultiThread;
|
||||
if(useMultiThread)
|
||||
{
|
||||
stopWorker = false;
|
||||
compressThread = new Thread(() => CompressAsync());
|
||||
compressThread.Start();
|
||||
}
|
||||
}
|
||||
|
||||
void InitBuffer(byte[] readBuffer, byte[] writeBuffer, byte[] outBuffer)
|
||||
{
|
||||
doubleBuffer = new DoubleBuffer(readBuffer ?? new byte[4 * MB], writeBuffer ?? new byte[4 * MB], Compress);
|
||||
this.outBuffer = outBuffer ?? new byte[wrapper.CompressBufferBound(writeBuffer.Length)];
|
||||
bfferWriter = new BufferWriter(doubleBuffer,this);
|
||||
}
|
||||
|
||||
public override void Flush()
|
||||
{
|
||||
doubleBuffer.SwapBuffer();
|
||||
if(useMultiThread)
|
||||
{
|
||||
doubleBuffer.WaitReadEnd();
|
||||
}
|
||||
lock (outBuffer)
|
||||
{
|
||||
outStream.Flush();
|
||||
}
|
||||
}
|
||||
|
||||
void Compress()
|
||||
{
|
||||
if (!useMultiThread)
|
||||
{
|
||||
Compress_Internal();
|
||||
}
|
||||
}
|
||||
|
||||
void Compress_Internal()
|
||||
{
|
||||
var consumeBuffer = doubleBuffer.ReadBegin();
|
||||
if (consumeBuffer.Length > 0)
|
||||
{
|
||||
lock (outBuffer)
|
||||
{
|
||||
long writeSize = 0;
|
||||
try
|
||||
{
|
||||
writeSize = wrapper.CompressUpdateEx(cctx, outBuffer, 0, consumeBuffer.Buffer, 0, consumeBuffer.Length);
|
||||
HandleError(writeSize);
|
||||
}
|
||||
finally
|
||||
{
|
||||
doubleBuffer.ReadEnd();
|
||||
}
|
||||
outStream.Write(outBuffer, 0, (int)writeSize);
|
||||
totalWrite += writeSize;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
doubleBuffer.ReadEnd();
|
||||
}
|
||||
}
|
||||
|
||||
void CompressAsync()
|
||||
{
|
||||
while(!stopWorker)
|
||||
{
|
||||
Compress_Internal();
|
||||
}
|
||||
}
|
||||
|
||||
public override int Read(byte[] buffer, int offset, int count)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override long Seek(long offset, SeekOrigin origin)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override void SetLength(long value)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
|
||||
public override void Write(byte[] buffer, int offset, int count)
|
||||
{
|
||||
BufferWriter.Write(buffer, offset, count);
|
||||
//var writeBuffer = doubleBuffer.writeBuffer;
|
||||
//int writeSize = writeBuffer.Write(buffer, offset, count);
|
||||
//while (count - writeSize > 0)
|
||||
//{
|
||||
// SwapBuffer(ref writeBuffer);
|
||||
// offset += writeSize;
|
||||
// count -= writeSize;
|
||||
// writeSize = writeBuffer.Write(buffer, offset, count);
|
||||
//}
|
||||
//inputSum += count;
|
||||
}
|
||||
|
||||
protected void FreeContext()
|
||||
{
|
||||
wrapper.CompressContextFree(cctx);
|
||||
cctx = IntPtr.Zero;
|
||||
}
|
||||
|
||||
bool closed = false;
|
||||
public override void Close()
|
||||
{
|
||||
if(!closed)
|
||||
{
|
||||
BufferWriter.Close();
|
||||
closed = true;
|
||||
//Console.WriteLine($"FLUSH");
|
||||
|
||||
Flush();
|
||||
|
||||
// try stop the worker
|
||||
stopWorker = true;
|
||||
doubleBuffer.SwapBuffer();
|
||||
|
||||
long size = wrapper.CompressEnd(cctx, outBuffer, outBuffer.Length);
|
||||
//Debug.Log($"End");
|
||||
outStream.Write(outBuffer, 0, (int)size);
|
||||
base.Close();
|
||||
}
|
||||
}
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
FreeContext();
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
|
||||
}
|
||||
148
CompressSave/Wrapper/DecompressionStream.cs
Normal file
148
CompressSave/Wrapper/DecompressionStream.cs
Normal file
@@ -0,0 +1,148 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace CompressSave.Wrapper;
|
||||
|
||||
public class DecompressionStream : Stream
|
||||
{
|
||||
public WrapperDefines wrapper;
|
||||
|
||||
public override bool CanRead => true;
|
||||
|
||||
public override bool CanSeek => false;
|
||||
|
||||
public override bool CanWrite => false;
|
||||
|
||||
public override long Length => inStream.Length;
|
||||
|
||||
public override long Position
|
||||
{ get => readPos;
|
||||
set
|
||||
{
|
||||
if (value < readPos)
|
||||
ResetStream();
|
||||
else
|
||||
value -= readPos;
|
||||
byte[] tmpBuffer = new byte[1024];
|
||||
while (value > 0)
|
||||
{
|
||||
value -= Read(tmpBuffer, 0, (int)(value < 1024 ? value : 1024));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Stream inStream;
|
||||
|
||||
IntPtr dctx = IntPtr.Zero;
|
||||
|
||||
readonly ByteSpan srcBuffer;
|
||||
readonly ByteSpan dcmpBuffer;
|
||||
private bool decompressFinish = false;
|
||||
readonly long startPos = 0;
|
||||
long readPos = 0; //sum of readlen
|
||||
|
||||
public DecompressionStream(WrapperDefines wrap, Stream inStream,int extraBufferSize = 512*1024)
|
||||
{
|
||||
this.wrapper = wrap;
|
||||
this.inStream = inStream;
|
||||
startPos = inStream.Position;
|
||||
srcBuffer = new ByteSpan(new byte[extraBufferSize]);
|
||||
int len = Fill();
|
||||
long expect = wrapper.DecompressBegin(ref dctx, srcBuffer.Buffer, ref len, out var blockSize);
|
||||
srcBuffer.Position += len;
|
||||
if (expect < 0) throw new Exception(expect.ToString());
|
||||
dcmpBuffer = new ByteSpan(new byte[blockSize]);
|
||||
}
|
||||
|
||||
public void ResetStream()
|
||||
{
|
||||
inStream.Seek(startPos, SeekOrigin.Begin);
|
||||
decompressFinish = false;
|
||||
srcBuffer.Clear();
|
||||
dcmpBuffer.Clear();
|
||||
wrapper.DecompressContextReset(dctx);
|
||||
readPos = 0;
|
||||
}
|
||||
|
||||
public int Fill()
|
||||
{
|
||||
int suplus = srcBuffer.Length - srcBuffer.Position;
|
||||
if (srcBuffer.Length> 0 && srcBuffer.Position >= suplus)
|
||||
{
|
||||
Array.Copy(srcBuffer, srcBuffer.Position, srcBuffer, 0, suplus);
|
||||
srcBuffer.Length -= srcBuffer.Position;
|
||||
srcBuffer.Position = 0;
|
||||
}
|
||||
if (srcBuffer.IdleCapacity > 0)
|
||||
{
|
||||
var readlen = inStream.Read(srcBuffer, srcBuffer.Length, srcBuffer.IdleCapacity);
|
||||
srcBuffer.Length += readlen;
|
||||
}
|
||||
return srcBuffer.Length - srcBuffer.Position;
|
||||
}
|
||||
|
||||
public override void Flush()
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
wrapper.DecompressEnd(dctx);
|
||||
dctx = IntPtr.Zero;
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
|
||||
public override int Read(byte[] buffer, int offset, int count)
|
||||
{
|
||||
int readlen = 0;
|
||||
while (count > (readlen += dcmpBuffer.Read(buffer, offset + readlen, count - readlen)) && !decompressFinish)
|
||||
{
|
||||
var buffSize = Fill();
|
||||
if (buffSize <= 0) return readlen;
|
||||
|
||||
var rt = wrapper.DecompressUpdateEx(dctx, dcmpBuffer, 0, dcmpBuffer.Capacity, srcBuffer, srcBuffer.Position,buffSize);
|
||||
if (rt.Expect < 0) throw new Exception(rt.Expect.ToString());
|
||||
if (rt.Expect == 0) decompressFinish = true;
|
||||
|
||||
srcBuffer.Position += (int)rt.ReadLen;
|
||||
dcmpBuffer.Position = 0;
|
||||
dcmpBuffer.Length = (int)rt.WriteLen;
|
||||
}
|
||||
readPos += readlen;
|
||||
return readlen;
|
||||
}
|
||||
|
||||
public int PeekByte()
|
||||
{
|
||||
if (dcmpBuffer.Length <= dcmpBuffer.Position)
|
||||
{
|
||||
var buffSize = Fill();
|
||||
if (buffSize <= 0) return -1;
|
||||
|
||||
var rt = wrapper.DecompressUpdateEx(dctx, dcmpBuffer, 0, dcmpBuffer.Capacity, srcBuffer, srcBuffer.Position, buffSize);
|
||||
if (rt.Expect < 0) throw new Exception(rt.Expect.ToString());
|
||||
if (rt.Expect == 0) decompressFinish = true;
|
||||
|
||||
srcBuffer.Position += (int)rt.ReadLen;
|
||||
dcmpBuffer.Position = 0;
|
||||
dcmpBuffer.Length = (int)rt.WriteLen;
|
||||
}
|
||||
return dcmpBuffer.Buffer[dcmpBuffer.Position];
|
||||
}
|
||||
|
||||
public override long Seek(long offset, SeekOrigin origin)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override void SetLength(long value)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override void Write(byte[] buffer, int offset, int count)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
131
CompressSave/Wrapper/DoubleBuffer.cs
Normal file
131
CompressSave/Wrapper/DoubleBuffer.cs
Normal file
@@ -0,0 +1,131 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
|
||||
namespace CompressSave.Wrapper;
|
||||
|
||||
public class ByteSpan
|
||||
{
|
||||
public byte[] Buffer { get; private set; }
|
||||
//public int Start;
|
||||
public int Length;
|
||||
public int Capacity;
|
||||
public int IdleCapacity => Capacity - Length;
|
||||
public int Position;
|
||||
|
||||
public ByteSpan(byte[] buffer)
|
||||
{
|
||||
Buffer = buffer;
|
||||
Capacity = Buffer.Length;
|
||||
}
|
||||
public void Clear()
|
||||
{
|
||||
Length = 0;
|
||||
Position = 0;
|
||||
}
|
||||
public int Write(byte[] src, int offset, int count)
|
||||
{
|
||||
int writeLen = Math.Min(Capacity - Length, count);
|
||||
Array.Copy(src, offset, Buffer, Length, writeLen);
|
||||
Length += writeLen;
|
||||
return writeLen;
|
||||
}
|
||||
|
||||
public int Read(byte[] dst, int offset, int count)
|
||||
{
|
||||
count = Math.Min(Length - Position, count);
|
||||
Array.Copy(Buffer, Position, dst, offset, count);
|
||||
Position += count;
|
||||
return count;
|
||||
}
|
||||
|
||||
public static implicit operator byte[](ByteSpan bs) => bs.Buffer;
|
||||
}
|
||||
|
||||
public struct ReadOnlySpan
|
||||
{
|
||||
public readonly int Length;
|
||||
public readonly byte[] Buffer;
|
||||
public int Position;
|
||||
|
||||
public ReadOnlySpan(byte[] buffer, int length)
|
||||
{
|
||||
Buffer = buffer;
|
||||
Length = length;
|
||||
Position = 0;
|
||||
}
|
||||
|
||||
public int Read(byte[] dst, int offset, int count)
|
||||
{
|
||||
count = Math.Min(Length - Position, count);
|
||||
Array.Copy(Buffer, Position, dst, offset, count);
|
||||
Position += count;
|
||||
return count;
|
||||
}
|
||||
|
||||
public static implicit operator byte[](ReadOnlySpan s) => s.Buffer;
|
||||
}
|
||||
|
||||
public class DoubleBuffer
|
||||
{
|
||||
public const int MB = 1024 * 1024;
|
||||
|
||||
public ByteSpan writeBuffer;
|
||||
public ByteSpan readBuffer;
|
||||
private ByteSpan midBuffer;
|
||||
private Action onReadBufferReady;
|
||||
|
||||
Semaphore readEnd = new Semaphore(1, 1);
|
||||
Semaphore writeEnd = new Semaphore(0, 1);
|
||||
|
||||
public DoubleBuffer(byte[] readBuffer, byte[] writeBuffer, Action onReadBufferReady)
|
||||
{
|
||||
this.onReadBufferReady = onReadBufferReady;
|
||||
this.midBuffer = new ByteSpan(readBuffer);
|
||||
this.writeBuffer = new ByteSpan(writeBuffer);
|
||||
}
|
||||
|
||||
public ByteSpan ReadBegin()
|
||||
{
|
||||
writeEnd.WaitOne();
|
||||
return readBuffer;
|
||||
}
|
||||
|
||||
public void ReadEnd()
|
||||
{
|
||||
readBuffer.Clear();
|
||||
midBuffer = readBuffer;
|
||||
readBuffer = null;
|
||||
readEnd.Release();
|
||||
}
|
||||
/// <summary>
|
||||
/// swap current write buffer to read and wait a new write buffer
|
||||
/// </summary>
|
||||
/// <returns> write buffer </returns>
|
||||
public ByteSpan SwapBuffer(bool triggerEvent = true)
|
||||
{
|
||||
var write = SwapBegin();
|
||||
SwapEnd();
|
||||
onReadBufferReady?.Invoke();
|
||||
return write;
|
||||
}
|
||||
|
||||
public void WaitReadEnd()
|
||||
{
|
||||
readEnd.WaitOne();
|
||||
readEnd.Release();
|
||||
}
|
||||
|
||||
public ByteSpan SwapBegin()
|
||||
{
|
||||
readEnd.WaitOne();
|
||||
readBuffer = writeBuffer;
|
||||
writeBuffer = midBuffer;
|
||||
midBuffer = null;
|
||||
return writeBuffer;
|
||||
}
|
||||
|
||||
public void SwapEnd()
|
||||
{
|
||||
writeEnd.Release();
|
||||
}
|
||||
}
|
||||
68
CompressSave/Wrapper/LZ4Wrapper.cs
Normal file
68
CompressSave/Wrapper/LZ4Wrapper.cs
Normal file
@@ -0,0 +1,68 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using MonoMod.Utils;
|
||||
|
||||
namespace CompressSave.Wrapper;
|
||||
|
||||
public class LZ4API: WrapperDefines
|
||||
{
|
||||
public static readonly bool Avaliable;
|
||||
|
||||
static LZ4API()
|
||||
{
|
||||
Avaliable = true;
|
||||
string assemblyPath = System.Reflection.Assembly.GetAssembly(typeof(LZ4API)).Location;
|
||||
string root = string.Empty;
|
||||
try
|
||||
{
|
||||
if (!string.IsNullOrEmpty(assemblyPath))
|
||||
{
|
||||
root = Path.GetDirectoryName(assemblyPath) ?? string.Empty;
|
||||
}
|
||||
|
||||
var map = new Dictionary<string, List<DynDllMapping>>
|
||||
{
|
||||
{
|
||||
"lz4wrap.dll", new List<DynDllMapping>
|
||||
{
|
||||
"lz4wrap.dll",
|
||||
"X64/lz4wrap.dll",
|
||||
"BepInEx/scripts/x64/lz4wrap.dll",
|
||||
Path.Combine(root, "X64/lz4wrap.dll"),
|
||||
Path.Combine(root, "lz4wrap.dll")
|
||||
}
|
||||
},
|
||||
};
|
||||
typeof(LZ4API).ResolveDynDllImports(map);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Avaliable = false;
|
||||
Console.WriteLine($"Error: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
public LZ4API()
|
||||
{
|
||||
CompressBufferBound = CompressBufferBound_;
|
||||
CompressBegin = CompressBegin_;
|
||||
CompressEnd = CompressEnd_;
|
||||
CompressUpdate = CompressUpdate_;
|
||||
CompressContextFree = CompressContextFree_;
|
||||
DecompressBegin = DecompressBegin_;
|
||||
DecompressEnd = DecompressEnd_;
|
||||
DecompressUpdate = DecompressUpdate_;
|
||||
DecompressContextReset = DecompressContextReset_;
|
||||
}
|
||||
|
||||
[DynDllImport(libraryName: "lz4wrap.dll", "CompressBufferBound")] protected static CompressBufferBoundFunc CompressBufferBound_;
|
||||
[DynDllImport(libraryName: "lz4wrap.dll", "CompressBegin")] protected static CompressBeginFunc CompressBegin_;
|
||||
[DynDllImport(libraryName: "lz4wrap.dll", "CompressEnd")] protected static CompressEndFunc CompressEnd_;
|
||||
[DynDllImport(libraryName: "lz4wrap.dll", "CompressUpdate")] protected static CompressUpdateFunc CompressUpdate_;
|
||||
[DynDllImport(libraryName: "lz4wrap.dll", "CompressContextFree")] protected static CompressContextFreeFunc CompressContextFree_;
|
||||
[DynDllImport(libraryName: "lz4wrap.dll", "DecompressBegin")] protected static DecompressBeginFunc DecompressBegin_;
|
||||
[DynDllImport(libraryName: "lz4wrap.dll", "DecompressEnd")] protected static DecompressEndFunc DecompressEnd_;
|
||||
[DynDllImport(libraryName: "lz4wrap.dll", "DecompressUpdate")] protected static DecompressUpdateFunc DecompressUpdate_;
|
||||
[DynDllImport(libraryName: "lz4wrap.dll", "DecompressContextReset")] protected static DecompressContextResetFunc DecompressContextReset_;
|
||||
}
|
||||
17
CompressSave/Wrapper/PeekableReader.cs
Normal file
17
CompressSave/Wrapper/PeekableReader.cs
Normal file
@@ -0,0 +1,17 @@
|
||||
using System.IO;
|
||||
|
||||
namespace CompressSave.Wrapper;
|
||||
|
||||
class PeekableReader : BinaryReader
|
||||
{
|
||||
DecompressionStream decompressionStream;
|
||||
public PeekableReader(DecompressionStream input) : base (input)
|
||||
{
|
||||
decompressionStream = input;
|
||||
}
|
||||
|
||||
public override int PeekChar()
|
||||
{
|
||||
return decompressionStream.PeekByte();
|
||||
}
|
||||
}
|
||||
64
CompressSave/Wrapper/WrapperDefines.cs
Normal file
64
CompressSave/Wrapper/WrapperDefines.cs
Normal file
@@ -0,0 +1,64 @@
|
||||
using System;
|
||||
|
||||
namespace CompressSave.Wrapper;
|
||||
|
||||
public struct DecompressStatus
|
||||
{
|
||||
public long WriteLen;
|
||||
public long ReadLen;
|
||||
public long Expect;
|
||||
}
|
||||
|
||||
public class WrapperDefines
|
||||
{
|
||||
public delegate long CompressBufferBoundFunc(long inBufferSize);
|
||||
public delegate long CompressBeginFunc(out IntPtr ctx, int compressionLevel, byte[] outBuff, long outCapacity, byte[] dictBuffer = null,
|
||||
long dictSize = 0);
|
||||
public delegate long CompressEndFunc(IntPtr ctx, byte[] dstBuffer, long dstCapacity);
|
||||
public delegate void CompressContextFreeFunc(IntPtr ctx);
|
||||
public delegate long DecompressBeginFunc(ref IntPtr pdctx, byte[] inBuffer, ref int inBufferSize, out int blockSize, byte[] dict = null, long dictSize = 0);
|
||||
public delegate long DecompressEndFunc(IntPtr dctx);
|
||||
public delegate void DecompressContextResetFunc(IntPtr dctx);
|
||||
protected unsafe delegate long CompressUpdateFunc(IntPtr ctx, byte* dstBuffer, long dstCapacity, byte* srcBuffer,
|
||||
long srcSize);
|
||||
protected unsafe delegate long DecompressUpdateFunc(IntPtr dctx, byte* dstBuffer, ref long dstCapacity, byte* srcBuffer,
|
||||
ref long srcSize);
|
||||
|
||||
public CompressBufferBoundFunc CompressBufferBound;
|
||||
public CompressBeginFunc CompressBegin;
|
||||
public CompressEndFunc CompressEnd;
|
||||
public CompressContextFreeFunc CompressContextFree;
|
||||
public DecompressBeginFunc DecompressBegin;
|
||||
public DecompressEndFunc DecompressEnd;
|
||||
public DecompressContextResetFunc DecompressContextReset;
|
||||
protected CompressUpdateFunc CompressUpdate;
|
||||
protected DecompressUpdateFunc DecompressUpdate;
|
||||
|
||||
public unsafe long CompressUpdateEx(IntPtr ctx, byte[] dstBuffer, long dstOffset, byte[] srcBuffer,
|
||||
long srcOffset, long srcLen)
|
||||
{
|
||||
fixed (byte* pdst = dstBuffer, psrc = srcBuffer)
|
||||
{
|
||||
return CompressUpdate(ctx, pdst + dstOffset, dstBuffer.Length - dstOffset, psrc + srcOffset,
|
||||
srcLen - srcOffset);
|
||||
}
|
||||
}
|
||||
|
||||
public unsafe DecompressStatus DecompressUpdateEx(IntPtr dctx, byte[] dstBuffer, int dstOffset, int dstCount,
|
||||
byte[] srcBuffer, long srcOffset, long count)
|
||||
{
|
||||
long dstLen = Math.Min(dstCount, dstBuffer.Length - dstOffset);
|
||||
long errCode;
|
||||
fixed (byte* pdst = dstBuffer, psrc = srcBuffer)
|
||||
{
|
||||
errCode = DecompressUpdate(dctx, pdst + dstOffset, ref dstLen, psrc + srcOffset, ref count);
|
||||
}
|
||||
|
||||
return new DecompressStatus
|
||||
{
|
||||
Expect = errCode,
|
||||
ReadLen = count,
|
||||
WriteLen = dstLen,
|
||||
};
|
||||
}
|
||||
}
|
||||
68
CompressSave/Wrapper/ZstdWrapper.cs
Normal file
68
CompressSave/Wrapper/ZstdWrapper.cs
Normal file
@@ -0,0 +1,68 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using MonoMod.Utils;
|
||||
|
||||
namespace CompressSave.Wrapper;
|
||||
|
||||
public class ZstdAPI: WrapperDefines
|
||||
{
|
||||
public static readonly bool Avaliable;
|
||||
|
||||
static ZstdAPI()
|
||||
{
|
||||
Avaliable = true;
|
||||
string assemblyPath = System.Reflection.Assembly.GetAssembly(typeof(ZstdAPI)).Location;
|
||||
string root = string.Empty;
|
||||
try
|
||||
{
|
||||
if (!string.IsNullOrEmpty(assemblyPath))
|
||||
{
|
||||
root = Path.GetDirectoryName(assemblyPath) ?? string.Empty;
|
||||
}
|
||||
|
||||
var map = new Dictionary<string, List<DynDllMapping>>
|
||||
{
|
||||
{
|
||||
"zstdwrap.dll", new List<DynDllMapping>
|
||||
{
|
||||
"zstdwrap.dll",
|
||||
"X64/zstdwrap.dll",
|
||||
"BepInEx/scripts/x64/zstdwrap.dll",
|
||||
Path.Combine(root, "X64/zstdwrap.dll"),
|
||||
Path.Combine(root, "zstdwrap.dll")
|
||||
}
|
||||
},
|
||||
};
|
||||
typeof(ZstdAPI).ResolveDynDllImports(map);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Avaliable = false;
|
||||
Console.WriteLine($"Error: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
public ZstdAPI()
|
||||
{
|
||||
CompressBufferBound = CompressBufferBound_;
|
||||
CompressBegin = CompressBegin_;
|
||||
CompressEnd = CompressEnd_;
|
||||
CompressUpdate = CompressUpdate_;
|
||||
CompressContextFree = CompressContextFree_;
|
||||
DecompressBegin = DecompressBegin_;
|
||||
DecompressEnd = DecompressEnd_;
|
||||
DecompressUpdate = DecompressUpdate_;
|
||||
DecompressContextReset = DecompressContextReset_;
|
||||
}
|
||||
|
||||
[DynDllImport(libraryName: "zstdwrap.dll", "CompressBufferBound")] protected static CompressBufferBoundFunc CompressBufferBound_;
|
||||
[DynDllImport(libraryName: "zstdwrap.dll", "CompressBegin")] protected static CompressBeginFunc CompressBegin_;
|
||||
[DynDllImport(libraryName: "zstdwrap.dll", "CompressEnd")] protected static CompressEndFunc CompressEnd_;
|
||||
[DynDllImport(libraryName: "zstdwrap.dll", "CompressUpdate")] protected static CompressUpdateFunc CompressUpdate_;
|
||||
[DynDllImport(libraryName: "zstdwrap.dll", "CompressContextFree")] protected static CompressContextFreeFunc CompressContextFree_;
|
||||
[DynDllImport(libraryName: "zstdwrap.dll", "DecompressBegin")] protected static DecompressBeginFunc DecompressBegin_;
|
||||
[DynDllImport(libraryName: "zstdwrap.dll", "DecompressEnd")] protected static DecompressEndFunc DecompressEnd_;
|
||||
[DynDllImport(libraryName: "zstdwrap.dll", "DecompressUpdate")] protected static DecompressUpdateFunc DecompressUpdate_;
|
||||
[DynDllImport(libraryName: "zstdwrap.dll", "DecompressContextReset")] protected static DecompressContextResetFunc DecompressContextReset_;
|
||||
}
|
||||
Reference in New Issue
Block a user