From 76faf2dd46c4ae8628acfe59fc500a3821bfc7b2 Mon Sep 17 00:00:00 2001 From: Florian Roks Date: Sat, 10 Aug 2024 21:18:45 +0200 Subject: [PATCH] refactoring read/write code to be more generic --- src/main/kotlin/dltcore/DltObjects.kt | 166 +++++++++--------- src/main/kotlin/dltcore/DltParser.kt | 131 +------------- src/main/kotlin/library/BinaryInputStream.kt | 22 +++ src/main/kotlin/library/BinaryOutputStream.kt | 11 ++ src/main/kotlin/library/ByteOrder.kt | 6 + .../jvm/ByteBufferBinaryInputStream.kt | 35 ++++ .../jvm/ByteBufferBinaryOutputStream.kt | 31 ++++ .../library/jvm/ByteBufferStreamExtensions.kt | 8 + .../library/jvm/LargeFileBinaryInputStream.kt | 83 +++++++++ 9 files changed, 286 insertions(+), 207 deletions(-) create mode 100644 src/main/kotlin/library/BinaryInputStream.kt create mode 100644 src/main/kotlin/library/BinaryOutputStream.kt create mode 100644 src/main/kotlin/library/ByteOrder.kt create mode 100644 src/main/kotlin/library/jvm/ByteBufferBinaryInputStream.kt create mode 100644 src/main/kotlin/library/jvm/ByteBufferBinaryOutputStream.kt create mode 100644 src/main/kotlin/library/jvm/ByteBufferStreamExtensions.kt create mode 100644 src/main/kotlin/library/jvm/LargeFileBinaryInputStream.kt diff --git a/src/main/kotlin/dltcore/DltObjects.kt b/src/main/kotlin/dltcore/DltObjects.kt index 1052fca..5c44fe1 100644 --- a/src/main/kotlin/dltcore/DltObjects.kt +++ b/src/main/kotlin/dltcore/DltObjects.kt @@ -1,7 +1,8 @@ package dltcore -import java.nio.ByteBuffer -import java.nio.ByteOrder +import library.BinaryInputStream +import library.BinaryOutputStream +import library.ByteOrder import java.nio.charset.Charset import java.time.Instant import kotlin.experimental.and @@ -20,7 +21,7 @@ public enum class DltStorageVersion(public val magicValue: Int) { public interface DltMessage { public fun getVersion(): DltStorageVersion - public fun write(bb: ByteBuffer) + public fun write(bb: BinaryOutputStream) } public class DltMessageV1( @@ -36,7 +37,7 @@ public class DltMessageV1( get() = extendedHeader?.messageTypeInfo - override fun write(bb: ByteBuffer) { + override fun write(bb: BinaryOutputStream) { storageHeader.write(bb) standardHeader.write(bb) extendedHeader?.write(bb) @@ -44,18 +45,18 @@ public class DltMessageV1( } public companion object { - public fun fromByteBuffer(buffer: ByteBuffer): DltMessageV1 { - val storageHeader = DltStorageHeaderV1.fromByteBuffer(buffer) - val standardHeader = DltStandardHeaderV1.fromByteBuffer(buffer) + public fun read(bis: BinaryInputStream): DltMessageV1 { + val storageHeader = DltStorageHeaderV1.read(bis) + val standardHeader = DltStandardHeaderV1.read(bis) val extendedHeader = if (standardHeader.useExtendedHeader) - DltExtendedHeaderV1.fromByteBuffer(buffer) + DltExtendedHeaderV1.read(bis) else null val payloadLength = standardHeader.len.toInt() - standardHeader.totalLength - (extendedHeader?.totalLength ?: 0) - val payload = DltPayload.fromByteBuffer( - buffer, + val payload = DltPayload.read( + bis, payloadLength, standardHeader.mostSignificantByteFirst, extendedHeader @@ -73,14 +74,14 @@ public class DltStorageHeaderV1( public val ecuIdText: String get() = ecuId.asStringValue() - public fun write(bb: ByteBuffer) { + public fun write(bb: BinaryOutputStream) { bb.order(ByteOrder.BIG_ENDIAN) - bb.putInt(DltStorageVersion.V1.magicValue) + bb.writeInt(DltStorageVersion.V1.magicValue) bb.order(ByteOrder.LITTLE_ENDIAN) - bb.putInt((timestampEpochSeconds and 0xFFFFFFFF).toInt()) - bb.putInt(timestampMicroseconds) - bb.putInt(ecuId) + bb.writeInt((timestampEpochSeconds and 0xFFFFFFFF).toInt()) + bb.writeInt(timestampMicroseconds) + bb.writeInt(ecuId) } public val utcTimestamp: Instant by lazy { @@ -88,13 +89,13 @@ public class DltStorageHeaderV1( } public companion object { - public fun fromByteBuffer(buffer: ByteBuffer): DltStorageHeaderV1 { + public fun read(buffer: BinaryInputStream): DltStorageHeaderV1 { // why is this little endian, when all other headers are big? buffer.order(ByteOrder.LITTLE_ENDIAN) - val timestampEpochSeconds = buffer.int.toUInt().toLong() - val timestampMicroseconds = buffer.int - val ecuId = buffer.int + val timestampEpochSeconds = buffer.readInt().toUInt().toLong() + val timestampMicroseconds = buffer.readInt() + val ecuId = buffer.readInt() return DltStorageHeaderV1(timestampEpochSeconds, timestampMicroseconds, ecuId) } } @@ -108,20 +109,20 @@ public class DltStandardHeaderV1( public val sessionId: Int?, public val timestamp: UInt?, ) { - public fun write(bb: ByteBuffer) { + public fun write(bb: BinaryOutputStream) { // The Standard Header and the Extended Header shall be in big endian format (MSB first). bb.order(ByteOrder.BIG_ENDIAN) - bb.put(htyp) - bb.put(mcnt.toByte()) - bb.putShort(len.toShort()) + bb.writeByte(htyp) + bb.writeByte(mcnt.toByte()) + bb.writeShort(len.toShort()) if (ecuId != null) { - bb.putInt(ecuId) + bb.writeInt(ecuId) } if (sessionId != null) { - bb.putInt(sessionId) + bb.writeInt(sessionId) } if (timestamp != null) { - bb.putInt(timestamp.toInt()) + bb.writeInt(timestamp.toInt()) } } @@ -159,17 +160,17 @@ public class DltStandardHeaderV1( public const val HEADER_TYPE_WSID: Byte = (1 shl 3).toByte() // with Session ID public const val HEADER_TYPE_WTMS: Byte = (1 shl 4).toByte() // with timestamp - public fun fromByteBuffer(buffer: ByteBuffer): DltStandardHeaderV1 { + public fun read(buffer: BinaryInputStream): DltStandardHeaderV1 { // The Standard Header and the Extended Header shall be in big endian format (MSB first). buffer.order(ByteOrder.BIG_ENDIAN) - val htyp = buffer.get() - val mcnt = buffer.get().toUByte() - val len = buffer.short.toUShort() + val htyp = buffer.readByte() + val mcnt = buffer.readByte().toUByte() + val len = buffer.readShort().toUShort() - val ecuId = if (htyp.and(HEADER_TYPE_WEID) > 0) buffer.int else null - val sid = if (htyp.and(HEADER_TYPE_WSID) > 0) buffer.int else null - val timestamp = if (htyp.and(HEADER_TYPE_WTMS) > 0) buffer.int.toUInt() else null + val ecuId = if (htyp.and(HEADER_TYPE_WEID) > 0) buffer.readInt() else null + val sid = if (htyp.and(HEADER_TYPE_WSID) > 0) buffer.readInt() else null + val timestamp = if (htyp.and(HEADER_TYPE_WTMS) > 0) buffer.readInt().toUInt() else null return DltStandardHeaderV1(htyp, mcnt, len, ecuId, sid, timestamp) } @@ -239,13 +240,13 @@ public class DltExtendedHeaderV1( public val apid: Int, // application id public val ctid: Int, // context id ) { - public fun write(bb: ByteBuffer) { + public fun write(bb: BinaryOutputStream) { bb.order(ByteOrder.BIG_ENDIAN) - bb.put(msin) - bb.put(noar) - bb.putInt(apid) - bb.putInt(ctid) + bb.writeByte(msin) + bb.writeByte(noar) + bb.writeInt(apid) + bb.writeInt(ctid) } public val isVerbose: Boolean @@ -274,14 +275,14 @@ public class DltExtendedHeaderV1( public const val MESSAGEINFO_MSTP: Byte = 0b1110.toByte() // message type public const val MESSAGEINFO_MTIN: Byte = 0xF0.toByte() // message type info - public fun fromByteBuffer(buffer: ByteBuffer): DltExtendedHeaderV1 { + public fun read(buffer: BinaryInputStream): DltExtendedHeaderV1 { // The Standard Header and the Extended Header shall be in big endian format (MSB first). buffer.order(ByteOrder.BIG_ENDIAN) - val msin = buffer.get() - val noar = buffer.get() - val apid = buffer.int - val ctid = buffer.int + val msin = buffer.readByte() + val noar = buffer.readByte() + val apid = buffer.readInt() + val ctid = buffer.readInt() return DltExtendedHeaderV1(msin, noar, apid, ctid) } } @@ -295,19 +296,18 @@ public class DltPayload( private val byteOrder = if (mostSignificantByteFirst) ByteOrder.BIG_ENDIAN else ByteOrder.LITTLE_ENDIAN public val logMessage: String by lazy { - val bb = ByteBuffer.wrap(data) + val bb = BinaryInputStream.wrap(data) bb.order(byteOrder) if (extendedHeader?.isVerbose != true) { - bb.int - val msg = ByteArray(data.size - 4) - bb.get(msg) + bb.readInt() + val msg = bb.readArray(data.size - 4) return@lazy String(msg) } else { val sb = StringBuilder() var len = data.size for (i in 0 until extendedHeader.noar) { - val arg = DltPayloadArgument.fromByteBuffer(bb, byteOrder) + val arg = DltPayloadArgument.read(bb, byteOrder) len -= arg.totalLength if (arg.variableName != null) { sb.append(arg.variableName) @@ -321,7 +321,7 @@ public class DltPayload( } } - public fun write(bb: ByteBuffer) { + public fun write(bb: BinaryOutputStream) { bb.order(byteOrder) bb.put(data) @@ -329,14 +329,13 @@ public class DltPayload( public companion object { - public fun fromByteBuffer( - bb: ByteBuffer, + public fun read( + bis: BinaryInputStream, len: Int, mostSignificantByteFirst: Boolean, extendedHeader: DltExtendedHeaderV1? ): DltPayload { - val data = ByteArray(len) - bb.get(data) + val data = bis.readArray(len) return DltPayload(data, mostSignificantByteFirst, extendedHeader) } } @@ -380,45 +379,44 @@ public abstract class DltPayloadArgument( public const val TYPEINFO_MASK_TYLE: Int = 0x7 public const val TYPEINFO_MASK_STRING_ENCODING: Int = 0x38000 - public fun getVariableName(typeInfo: Int, bb: ByteBuffer): String? { + public fun getVariableName(typeInfo: Int, bb: BinaryInputStream): String? { if (typeInfo and TYPEINFO_VARI > 0) { - val len = bb.short.toUShort().toInt() - val nameArray = ByteArray(len) - bb.get(nameArray) + val len = bb.readShort().toUShort().toInt() + val nameArray = bb.readArray(len) return String(nameArray, 0, len - 1, Charsets.US_ASCII) } return null } - public fun fromByteBuffer(bb: ByteBuffer, byteOrder: ByteOrder): DltPayloadArgument { + public fun read(bb: BinaryInputStream, byteOrder: ByteOrder): DltPayloadArgument { bb.order(ByteOrder.LITTLE_ENDIAN) - val arTypeInfo = bb.int + val arTypeInfo = bb.readInt() val argType = DltPayloadArgumentType.getByTypeInfo(arTypeInfo) return when (argType) { - DltPayloadArgumentType.STRG -> DltPayloadArgumentString.fromByteBuffer(arTypeInfo, bb, byteOrder) - DltPayloadArgumentType.RAWD -> DltPayloadArgumentRawData.fromByteBuffer(arTypeInfo, bb, byteOrder) - DltPayloadArgumentType.UINT -> DltPayloadArgumentNumber.fromByteBuffer( + DltPayloadArgumentType.STRG -> DltPayloadArgumentString.read(arTypeInfo, bb, byteOrder) + DltPayloadArgumentType.RAWD -> DltPayloadArgumentRawData.read(arTypeInfo, bb, byteOrder) + DltPayloadArgumentType.UINT -> DltPayloadArgumentNumber.read( argType, arTypeInfo, bb, byteOrder ) - DltPayloadArgumentType.SINT -> DltPayloadArgumentNumber.fromByteBuffer( + DltPayloadArgumentType.SINT -> DltPayloadArgumentNumber.read( argType, arTypeInfo, bb, byteOrder ) - DltPayloadArgumentType.FLOA -> DltPayloadArgumentNumber.fromByteBuffer( + DltPayloadArgumentType.FLOA -> DltPayloadArgumentNumber.read( argType, arTypeInfo, bb, byteOrder ) - DltPayloadArgumentType.BOOL -> DltPayloadArgumentBool.fromByteBuffer( + DltPayloadArgumentType.BOOL -> DltPayloadArgumentBool.read( arTypeInfo, bb, byteOrder @@ -435,7 +433,7 @@ public class DltPayloadArgumentBool( variableName: String?, ) : DltPayloadArgument(DltPayloadArgumentType.BOOL, variableName) { public companion object { - public fun fromByteBuffer(typeInfo: Int, bb: ByteBuffer, byteOrder: ByteOrder): DltPayloadArgumentBool { + public fun read(typeInfo: Int, bb: BinaryInputStream, byteOrder: ByteOrder): DltPayloadArgumentBool { bb.order(byteOrder) val variableName = getVariableName(typeInfo, bb) @@ -443,7 +441,7 @@ public class DltPayloadArgumentBool( if (lenInfo != 1) { throw UnsupportedOperationException("Unsupported length info $lenInfo") } - return DltPayloadArgumentBool(bb.get() != 0x0.toByte(), variableName) + return DltPayloadArgumentBool(bb.readByte() != 0x0.toByte(), variableName) } } @@ -466,13 +464,12 @@ public class DltPayloadArgumentString( data public companion object { - public fun fromByteBuffer(typeInfo: Int, bb: ByteBuffer, byteOrder: ByteOrder): DltPayloadArgumentString { + public fun read(typeInfo: Int, bb: BinaryInputStream, byteOrder: ByteOrder): DltPayloadArgumentString { bb.order(byteOrder) val variableName = getVariableName(typeInfo, bb) - val len = bb.short.toUShort().toInt() - val data = ByteArray(len) - bb.get(data) + val len = bb.readShort().toUShort().toInt() + val data = bb.readArray(len) val charset = when (val encoding = (typeInfo and TYPEINFO_MASK_STRING_ENCODING) shr 15) { 0 -> Charsets.US_ASCII 1 -> Charsets.UTF_8 @@ -506,13 +503,12 @@ public class DltPayloadArgumentRawData( public companion object { @OptIn(ExperimentalStdlibApi::class) - public fun fromByteBuffer(typeInfo: Int, bb: ByteBuffer, byteOrder: ByteOrder): DltPayloadArgumentRawData { + public fun read(typeInfo: Int, bb: BinaryInputStream, byteOrder: ByteOrder): DltPayloadArgumentRawData { bb.order(byteOrder) val variableName = getVariableName(typeInfo, bb) - val len = bb.short.toUShort().toInt() - val data = ByteArray(len) - bb.get(data) + val len = bb.readShort().toUShort().toInt() + val data = bb.readArray(len) val s = data.toHexString(DltRawHexFormat) return DltPayloadArgumentRawData(s, variableName) @@ -532,10 +528,10 @@ public class DltPayloadArgumentNumber( get() = len public companion object { - public fun fromByteBuffer( + public fun read( type: DltPayloadArgumentType, typeInfo: Int, - bb: ByteBuffer, + bb: BinaryInputStream, byteOrder: ByteOrder ): DltPayloadArgumentNumber { bb.order(byteOrder) @@ -557,22 +553,22 @@ public class DltPayloadArgumentNumber( val number = when (type) { DltPayloadArgumentType.FLOA -> when (lenInBytes) { 2 -> throw UnsupportedOperationException("Half precision floats aren't supported") - 4 -> Float.fromBits(bb.int) - 8 -> Double.fromBits(bb.long) + 4 -> Float.fromBits(bb.readInt()) + 8 -> Double.fromBits(bb.readLong()) else -> throw UnsupportedOperationException("Unsupported float len $lenInfo") } DltPayloadArgumentType.SINT -> when (lenInBytes) { - 2 -> bb.short - 4 -> bb.int - 8 -> bb.long + 2 -> bb.readShort() + 4 -> bb.readInt() + 8 -> bb.readLong() else -> throw UnsupportedOperationException("Unsupported sint len $lenInBytes") } DltPayloadArgumentType.UINT -> when (lenInBytes) { - 2 -> bb.short - 4 -> bb.int - 8 -> bb.long + 2 -> bb.readShort() + 4 -> bb.readInt() + 8 -> bb.readLong() else -> throw UnsupportedOperationException("Unsupported uint len $lenInBytes") } diff --git a/src/main/kotlin/dltcore/DltParser.kt b/src/main/kotlin/dltcore/DltParser.kt index e9e023c..c0fea35 100644 --- a/src/main/kotlin/dltcore/DltParser.kt +++ b/src/main/kotlin/dltcore/DltParser.kt @@ -1,15 +1,7 @@ package dltcore -import java.nio.ByteBuffer -import java.nio.ByteOrder -import java.nio.channels.FileChannel -import java.nio.file.Path -import java.nio.file.StandardOpenOption -import java.util.* -import java.util.stream.Stream -import java.util.stream.StreamSupport -import kotlin.io.path.fileSize -import kotlin.math.min +import library.BinaryInputStream +import library.ByteOrder public data class DltReadStatus( val index: Long, @@ -22,102 +14,14 @@ public data class DltReadStatus( val dltMessage: DltMessage?, ) -private const val OVERLAP = 10_000_000 - -private class LargeFileBufferChooser(val path: Path) : AutoCloseable { - private lateinit var currentBuffer: ByteBuffer - - private val fileSize = path.fileSize() - private var fileChannel: FileChannel = FileChannel.open(path, StandardOpenOption.READ) - private var absolutePosition = -1L - private var bufferIndex = 0 - - val buffer: ByteBuffer - get() { - if (absolutePosition == -1L) { - absolutePosition = 0 - currentBuffer = fileChannel.map( - FileChannel.MapMode.READ_ONLY, - absolutePosition, - min(fileSize, Integer.MAX_VALUE.toLong()) - ) - bufferIndex = 0 - return currentBuffer - } - val relativePosition = currentBuffer.position() - if (relativePosition >= (Integer.MAX_VALUE - OVERLAP)) { - absolutePosition += relativePosition.toLong() - currentBuffer = fileChannel.map( - FileChannel.MapMode.READ_ONLY, - absolutePosition, - min(fileSize - absolutePosition, Integer.MAX_VALUE.toLong()) - ) - } - return currentBuffer - } - - override fun close() { - fileChannel.close() - } -} - -private class DltMessageIteratorPath(private val largeFile: LargeFileBufferChooser) : Iterator { - private var index: Long = 0 - private var successCount: Long = 0 - private var errorCount: Long = 0 - private val totalSize = largeFile.path.fileSize() - - private fun parseDltMessage(buffer: ByteBuffer, version: DltStorageVersion): DltMessage = - when (version) { - DltStorageVersion.V1 -> DltMessageV1.fromByteBuffer(buffer) - DltStorageVersion.V2 -> throw UnsupportedOperationException("not supported yet") - } - - override fun hasNext(): Boolean { - val buffer = largeFile.buffer - return buffer.hasRemaining() - } - - override fun next(): DltReadStatus { - val buffer = largeFile.buffer - buffer.order(ByteOrder.BIG_ENDIAN) - if (buffer.hasRemaining()) { - val message = try { - val magic = buffer.int - val version = DltStorageVersion.getByMagic(magic) - parseDltMessage(buffer, version) - } catch (e: RuntimeException) { - errorCount++ - throw RuntimeException( - "Error while parsing message at file position ${buffer.position()}: ${e.message}", - e - ) - } - successCount++ - val progress = buffer.position().toFloat() / totalSize.toFloat() - return DltReadStatus( - index = index++, - filePosition = buffer.position().toLong(), - fileSize = totalSize, - progress = progress, - progressText = "Parsing file", - errorCount = errorCount, - successCount = successCount, - dltMessage = message - ) - } - throw RuntimeException("No more data available, but next() was called on iterator") - } -} - -private class DltMessageIterator(val buffer: ByteBuffer, val totalSize: Long?) : Iterator { +private class DltMessageIterator(val buffer: BinaryInputStream, val totalSize: Long?) : Iterator { private var index: Long = 0 private var successCount: Long = 0 private var errorCount: Long = 0 - private fun parseDltMessage(buffer: ByteBuffer, version: DltStorageVersion): DltMessage = + private fun parseDltMessage(buffer: BinaryInputStream, version: DltStorageVersion): DltMessage = when (version) { - DltStorageVersion.V1 -> DltMessageV1.fromByteBuffer(buffer) + DltStorageVersion.V1 -> DltMessageV1.read(buffer) DltStorageVersion.V2 -> throw UnsupportedOperationException("not supported yet") } @@ -128,7 +32,7 @@ private class DltMessageIterator(val buffer: ByteBuffer, val totalSize: Long?) : buffer.order(ByteOrder.BIG_ENDIAN) if (buffer.hasRemaining()) { val message = try { - val magic = buffer.int + val magic = buffer.readInt() val version = DltStorageVersion.getByMagic(magic) parseDltMessage(buffer, version) } catch (e: RuntimeException) { @@ -146,7 +50,7 @@ private class DltMessageIterator(val buffer: ByteBuffer, val totalSize: Long?) : } return DltReadStatus( index = index++, - filePosition = buffer.position().toLong(), + filePosition = buffer.position(), fileSize = totalSize, progress = progress, progressText = "Parsing file", @@ -162,24 +66,7 @@ private class DltMessageIterator(val buffer: ByteBuffer, val totalSize: Long?) : public class DltMessageParser private constructor() { public companion object { - public fun parseBuffer(buffer: ByteBuffer, totalSize: Long?): Stream = - StreamSupport.stream( - Spliterators.spliteratorUnknownSize( - DltMessageIterator(buffer, totalSize), - Spliterator.NONNULL or Spliterator.ORDERED - ), false - ) - - public fun parseFile(path: Path): Stream { - val fb = LargeFileBufferChooser(path) - return StreamSupport.stream( - Spliterators.spliteratorUnknownSize( - DltMessageIteratorPath(fb), - Spliterator.NONNULL or Spliterator.ORDERED - ), false - ).onClose { - fb.close() - } - } + public fun parseBuffer(buffer: BinaryInputStream, totalSize: Long?): Sequence = + DltMessageIterator(buffer, totalSize).asSequence() } } diff --git a/src/main/kotlin/library/BinaryInputStream.kt b/src/main/kotlin/library/BinaryInputStream.kt new file mode 100644 index 0000000..30fc3b0 --- /dev/null +++ b/src/main/kotlin/library/BinaryInputStream.kt @@ -0,0 +1,22 @@ +package library + +import library.jvm.ByteBufferBinaryInputStream +import java.nio.ByteBuffer + +public interface BinaryInputStream { + public fun order(order: ByteOrder) + public fun hasRemaining(): Boolean + public fun position(): Long + + public fun readByte(): Byte + public fun readShort(): Short + public fun readInt(): Int + public fun readLong(): Long + + public fun readArray(len: Int): ByteArray + + public companion object { + public fun wrap(array: ByteArray): BinaryInputStream = + ByteBufferBinaryInputStream(ByteBuffer.wrap(array)) + } +} diff --git a/src/main/kotlin/library/BinaryOutputStream.kt b/src/main/kotlin/library/BinaryOutputStream.kt new file mode 100644 index 0000000..e1dcb6e --- /dev/null +++ b/src/main/kotlin/library/BinaryOutputStream.kt @@ -0,0 +1,11 @@ +package library + +public interface BinaryOutputStream { + public fun order(order: ByteOrder) + public fun put(data: ByteArray) + + public fun writeByte(value: Byte) + public fun writeShort(value: Short) + public fun writeInt(value: Int) + public fun writeLong(value: Long) +} diff --git a/src/main/kotlin/library/ByteOrder.kt b/src/main/kotlin/library/ByteOrder.kt new file mode 100644 index 0000000..35ed80d --- /dev/null +++ b/src/main/kotlin/library/ByteOrder.kt @@ -0,0 +1,6 @@ +package library + +public enum class ByteOrder { + BIG_ENDIAN, + LITTLE_ENDIAN +} diff --git a/src/main/kotlin/library/jvm/ByteBufferBinaryInputStream.kt b/src/main/kotlin/library/jvm/ByteBufferBinaryInputStream.kt new file mode 100644 index 0000000..1c5d602 --- /dev/null +++ b/src/main/kotlin/library/jvm/ByteBufferBinaryInputStream.kt @@ -0,0 +1,35 @@ +package library.jvm + +import library.BinaryInputStream +import library.ByteOrder +import java.nio.ByteBuffer + +public class ByteBufferBinaryInputStream(private val buffer: ByteBuffer) : BinaryInputStream { + override fun order(order: ByteOrder) { + buffer.order(order.asByteBufferByteOrder()) + } + + override fun hasRemaining(): Boolean = + buffer.hasRemaining() + + override fun position(): Long = + buffer.position().toLong() + + override fun readByte(): Byte = + buffer.get() + + override fun readShort(): Short = + buffer.getShort() + + override fun readInt(): Int = + buffer.getInt() + + override fun readLong(): Long = + buffer.getLong() + + override fun readArray(len: Int): ByteArray { + val data = ByteArray(len) + buffer.get(data) + return data + } +} diff --git a/src/main/kotlin/library/jvm/ByteBufferBinaryOutputStream.kt b/src/main/kotlin/library/jvm/ByteBufferBinaryOutputStream.kt new file mode 100644 index 0000000..e809890 --- /dev/null +++ b/src/main/kotlin/library/jvm/ByteBufferBinaryOutputStream.kt @@ -0,0 +1,31 @@ +package library.jvm + +import library.BinaryOutputStream +import library.ByteOrder +import java.nio.ByteBuffer + +public class ByteBufferBinaryOutputStream(private val buffer: ByteBuffer) : BinaryOutputStream { + override fun order(order: ByteOrder) { + buffer.order(order.asByteBufferByteOrder()) + } + + override fun put(data: ByteArray) { + buffer.put(data) + } + + override fun writeByte(value: Byte) { + buffer.put(value) + } + + override fun writeShort(value: Short) { + buffer.putShort(value) + } + + override fun writeInt(value: Int) { + buffer.putInt(value) + } + + override fun writeLong(value: Long) { + buffer.putLong(value) + } +} diff --git a/src/main/kotlin/library/jvm/ByteBufferStreamExtensions.kt b/src/main/kotlin/library/jvm/ByteBufferStreamExtensions.kt new file mode 100644 index 0000000..77da5f0 --- /dev/null +++ b/src/main/kotlin/library/jvm/ByteBufferStreamExtensions.kt @@ -0,0 +1,8 @@ +package library.jvm + +import library.ByteOrder + +public fun ByteOrder.asByteBufferByteOrder(): java.nio.ByteOrder = when (this) { + ByteOrder.LITTLE_ENDIAN -> java.nio.ByteOrder.LITTLE_ENDIAN + ByteOrder.BIG_ENDIAN -> java.nio.ByteOrder.BIG_ENDIAN +} diff --git a/src/main/kotlin/library/jvm/LargeFileBinaryInputStream.kt b/src/main/kotlin/library/jvm/LargeFileBinaryInputStream.kt new file mode 100644 index 0000000..3950e1e --- /dev/null +++ b/src/main/kotlin/library/jvm/LargeFileBinaryInputStream.kt @@ -0,0 +1,83 @@ +package library.jvm + +import dltcore.DltMessageParser +import dltcore.DltReadStatus +import library.BinaryInputStream +import library.ByteOrder +import java.nio.channels.FileChannel +import java.nio.file.Path +import java.nio.file.StandardOpenOption +import kotlin.io.path.fileSize +import kotlin.math.min + +public fun DltMessageParser.Companion.parseFile(path: Path): Sequence { + val bis = LargeFileByteBufferInputStream(path) + return parseBuffer(bis, path.fileSize()) +} + +private const val OVERLAP = 10_000_000 + +private class LargeFileByteBufferInputStream(path: Path) : BinaryInputStream { + private lateinit var currentInputStream: BinaryInputStream + + private val fileSize = path.fileSize() + private var fileChannel: FileChannel = FileChannel.open(path, StandardOpenOption.READ) + private var absolutePosition = -1L + private var bufferIndex = 0 + + private val buffer: BinaryInputStream + get() { + if (absolutePosition == -1L) { + absolutePosition = 0 + val buffer = fileChannel.map( + FileChannel.MapMode.READ_ONLY, + absolutePosition, + min(fileSize, Integer.MAX_VALUE.toLong()) + ) + currentInputStream = ByteBufferBinaryInputStream(buffer) + bufferIndex = 0 + return currentInputStream + } + val relativePosition = currentInputStream.position() + if (relativePosition >= (Integer.MAX_VALUE - OVERLAP)) { + absolutePosition += relativePosition + val buffer = fileChannel.map( + FileChannel.MapMode.READ_ONLY, + absolutePosition, + min(fileSize - absolutePosition, Integer.MAX_VALUE.toLong()) + ) + currentInputStream = ByteBufferBinaryInputStream(buffer) + } + return currentInputStream + } + + override fun order(order: ByteOrder) { + buffer.order(order) + } + + override fun hasRemaining(): Boolean { + val remaining = buffer.hasRemaining() + if (!remaining) { + fileChannel.close() + } + return remaining + } + + override fun position(): Long = + absolutePosition + currentInputStream.position() + + override fun readByte(): Byte = + buffer.readByte() + + override fun readShort(): Short = + buffer.readShort() + + override fun readInt(): Int = + buffer.readInt() + + override fun readLong(): Long = + buffer.readLong() + + override fun readArray(len: Int): ByteArray = + buffer.readArray(len) +}