Skip to content

Commit

Permalink
Merge pull request #448 from JetBrains/lazy-serializers-usov
Browse files Browse the repository at this point in the history 
Lazy serializers
  • Loading branch information
@Iliya-usov
Iliya-usov authored Nov 6, 2023
2 parents 571a9c4 + 5da9c7c commit 63d16a6
Show file tree
Hide file tree
Showing 24 changed files with 404 additions and 164 deletions.
8 changes: 7 additions & 1 deletion rd-kt/rd-core/src/main/kotlin/com/jetbrains/rd/util/hash/IncrementalHash.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,10 @@ class IncrementalHash64(r: Long = 0) {
fun mix(mix: String?) = IncrementalHash64(mix?.fold(result) { acc, c -> acc*31 + c.toInt()} ?: result)
fun<T> mix(collection : Iterable<T>, map : (T) -> String) = collection.fold(this) {acc, elt -> acc.mix(map(elt))}
val result: Long = r
}
}


//PLEASE DO NOT CHANGE IT!!! IT'S EXACTLY THE SAME ON C# SIDE
fun String?.getPlatformIndependentHash(initial: Long = 19L) : Long = this?.fold(initial) { acc, c -> acc*31 + c.toInt()} ?:0
fun Int.getPlatformIndependentHash(initial: Long = 19L) : Long = initial*31 + (this + 1)
fun Long.getPlatformIndependentHash(initial: Long = 19L) : Long = initial*31 + (this + 1)
78 changes: 42 additions & 36 deletions rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/FrameworkMarshallers.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package com.jetbrains.rd.framework
import com.jetbrains.rd.framework.impl.RdSecureString
import com.jetbrains.rd.util.*
import kotlin.reflect.*
import kotlin.time.Duration

open class UniversalMarshaller<T : Any>(
override val _type: KClass<*>,
Expand All @@ -27,13 +28,18 @@ open class DelegatedMarshaller<TFrom : Any, TTo : Any>(marshaller: IMarshaller<T
)

object FrameworkMarshallers {
inline fun <reified T : Any> create(crossinline reader: (AbstractBuffer) -> T, crossinline writer: (AbstractBuffer, T) -> Unit, predefinedId: Int? = null): UniversalMarshaller<T> {
return UniversalMarshaller(T::class, { _, stream -> reader(stream) }, { _, stream, v -> writer(stream, v) }, predefinedId)
@Deprecated("Use an overload without inlining", ReplaceWith("create(T::class, reader, writer, predefinedId)", "com.jetbrains.rd.framework.FrameworkMarshallers.create"))
inline fun <reified T : Any> create(noinline reader: (AbstractBuffer) -> T, noinline writer: (AbstractBuffer, T) -> Unit, predefinedId: Int? = null): UniversalMarshaller<T> {
return create(T::class, reader, writer, predefinedId)
}


fun <T : Any> create(clazz: KClass<T>, reader: (AbstractBuffer) -> T, writer: (AbstractBuffer, T) -> Unit, predefinedId: Int? = null): UniversalMarshaller<T> {
return UniversalMarshaller(clazz, { _, stream -> reader(stream) }, { _, stream, v -> writer(stream, v) }, predefinedId)
}


inline fun <reified T : Enum<T>> enum(): UniversalMarshaller<T> {
return create({ it.readEnum<T>() }, { stream, x -> stream.writeEnum(x) })
return create(T::class, { it.readEnum<T>() }, { stream, x -> stream.writeEnum(x) } )
}

inline fun <reified T : Enum<T>> enumSet(): UniversalMarshaller<EnumSet<T>> {
Expand All @@ -50,21 +56,21 @@ object FrameworkMarshallers {
}


val Int8 : IMarshaller<Byte> = create(AbstractBuffer::readByte, AbstractBuffer::writeByte, 1)
val Int16: IMarshaller<Short> = create(AbstractBuffer::readShort, AbstractBuffer::writeShort, 2)
val Int32: IMarshaller<Int> = create(AbstractBuffer::readInt, AbstractBuffer::writeInt, 3)
val Int64: IMarshaller<Long> = create(AbstractBuffer::readLong, AbstractBuffer::writeLong, 4)
val Int8 : IMarshaller<Byte> = create(kotlin.Byte::class, AbstractBuffer::readByte, AbstractBuffer::writeByte, 1)
val Int16: IMarshaller<Short> = create(kotlin.Short::class, AbstractBuffer::readShort, AbstractBuffer::writeShort, 2)
val Int32: IMarshaller<Int> = create(kotlin.Int::class, AbstractBuffer::readInt, AbstractBuffer::writeInt, 3)
val Int64: IMarshaller<Long> = create(kotlin.Long::class, AbstractBuffer::readLong, AbstractBuffer::writeLong, 4)

val Float: IMarshaller<Float> = create(AbstractBuffer::readFloat, AbstractBuffer::writeFloat, 5)
val Double: IMarshaller<Double> = create(AbstractBuffer::readDouble, AbstractBuffer::writeDouble, 6)
val Char: IMarshaller<Char> = create(AbstractBuffer::readChar, AbstractBuffer::writeChar, 7)
val Bool: IMarshaller<Boolean> = create(AbstractBuffer::readBoolean, AbstractBuffer::writeBoolean, 8)
val Float: IMarshaller<Float> = create(kotlin.Float::class, AbstractBuffer::readFloat, AbstractBuffer::writeFloat, 5)
val Double: IMarshaller<Double> = create(kotlin.Double::class, AbstractBuffer::readDouble, AbstractBuffer::writeDouble, 6)
val Char: IMarshaller<Char> = create(kotlin.Char::class, AbstractBuffer::readChar, AbstractBuffer::writeChar, 7)
val Bool: IMarshaller<Boolean> = create(Boolean::class, AbstractBuffer::readBoolean, AbstractBuffer::writeBoolean, 8)

//empty
val Void: IMarshaller<Unit> = create({ }, { _, _ -> }, 9)
val Void: IMarshaller<Unit> = create(Unit::class, { }, { _, _ -> }, 9)

//normal string
val String: UniversalMarshaller<String> = create(AbstractBuffer::readString, AbstractBuffer::writeString, 10)
val String: UniversalMarshaller<String> = create(kotlin.String::class, AbstractBuffer::readString, AbstractBuffer::writeString, 10)

//aliases
val Byte = Int8
Expand All @@ -74,48 +80,48 @@ object FrameworkMarshallers {


//jvm-based
val Guid: UniversalMarshaller<UUID> = create({ it.readUuid() }, AbstractBuffer::writeUuid, 11)
val DateTime: UniversalMarshaller<Date> = create(AbstractBuffer::readDateTime, AbstractBuffer::writeDateTime, 12)
val Uri: UniversalMarshaller<URI> = create({ it.readUri() }, AbstractBuffer::writeUri, 13)
var TimeSpan: UniversalMarshaller<kotlin.time.Duration> = create(AbstractBuffer::readTimeSpan, AbstractBuffer::writeTimeSpan, 14)
val Guid: UniversalMarshaller<UUID> = create(java.util.UUID::class, { it.readUuid() }, AbstractBuffer::writeUuid, 11)
val DateTime: UniversalMarshaller<Date> = create(java.util.Date::class, AbstractBuffer::readDateTime, AbstractBuffer::writeDateTime, 12)
val Uri: UniversalMarshaller<URI> = create(java.net.URI::class, { it.readUri() }, AbstractBuffer::writeUri, 13)
var TimeSpan: UniversalMarshaller<Duration> = create(Duration::class, AbstractBuffer::readTimeSpan, AbstractBuffer::writeTimeSpan, 14)

//rdId
val RdId: IMarshaller<RdId> = create(AbstractBuffer::readRdId, AbstractBuffer::writeRdId, 15)
val RdId: IMarshaller<RdId> = create(com.jetbrains.rd.framework.RdId::class, AbstractBuffer::readRdId, AbstractBuffer::writeRdId, 15)

//string for passwords
val SecureString: IMarshaller<RdSecureString> = create({ RdSecureString(it.readString()) }, { buf, str -> buf.writeString(str.contents) }, 16)
val SecureString: IMarshaller<RdSecureString> = create(RdSecureString::class, { RdSecureString(it.readString()) }, { buf, str -> buf.writeString(str.contents) }, 16)

//arrays
val ByteArray: UniversalMarshaller<ByteArray> = create(AbstractBuffer::readByteArray, AbstractBuffer::writeByteArray, 31)
val ShortArray: UniversalMarshaller<ShortArray> = create(AbstractBuffer::readShortArray, AbstractBuffer::writeShortArray, 32)
val IntArray: UniversalMarshaller<IntArray> = create(AbstractBuffer::readIntArray, AbstractBuffer::writeIntArray, 33)
val LongArray: UniversalMarshaller<LongArray> = create(AbstractBuffer::readLongArray, AbstractBuffer::writeLongArray, 34)
val ByteArray: UniversalMarshaller<ByteArray> = create(kotlin.ByteArray::class, AbstractBuffer::readByteArray, AbstractBuffer::writeByteArray, 31)
val ShortArray: UniversalMarshaller<ShortArray> = create(kotlin.ShortArray::class, AbstractBuffer::readShortArray, AbstractBuffer::writeShortArray, 32)
val IntArray: UniversalMarshaller<IntArray> = create(kotlin.IntArray::class, AbstractBuffer::readIntArray, AbstractBuffer::writeIntArray, 33)
val LongArray: UniversalMarshaller<LongArray> = create(kotlin.LongArray::class, AbstractBuffer::readLongArray, AbstractBuffer::writeLongArray, 34)

val FloatArray: UniversalMarshaller<FloatArray> = create(AbstractBuffer::readFloatArray, AbstractBuffer::writeFloatArray, 35)
val DoubleArray: UniversalMarshaller<DoubleArray> = create(AbstractBuffer::readDoubleArray, AbstractBuffer::writeDoubleArray, 36)
val FloatArray: UniversalMarshaller<FloatArray> = create(kotlin.FloatArray::class, AbstractBuffer::readFloatArray, AbstractBuffer::writeFloatArray, 35)
val DoubleArray: UniversalMarshaller<DoubleArray> = create(kotlin.DoubleArray::class, AbstractBuffer::readDoubleArray, AbstractBuffer::writeDoubleArray, 36)

val CharArray: UniversalMarshaller<CharArray> = create(AbstractBuffer::readCharArray, AbstractBuffer::writeCharArray, 37)
val BooleanArray: UniversalMarshaller<BooleanArray> = create(AbstractBuffer::readBooleanArray, AbstractBuffer::writeBooleanArray, 38)
val CharArray: UniversalMarshaller<CharArray> = create(kotlin.CharArray::class, AbstractBuffer::readCharArray, AbstractBuffer::writeCharArray, 37)
val BooleanArray: UniversalMarshaller<BooleanArray> = create(kotlin.BooleanArray::class, AbstractBuffer::readBooleanArray, AbstractBuffer::writeBooleanArray, 38)


//unsigned
@ExperimentalUnsignedTypes
val UByte : IMarshaller<UByte> = create(AbstractBuffer::readUByte, AbstractBuffer::writeUByte, 41)
val UByte : IMarshaller<UByte> = create(kotlin.UByte::class, AbstractBuffer::readUByte, AbstractBuffer::writeUByte, 41)
@ExperimentalUnsignedTypes
val UShort: IMarshaller<UShort> = create(AbstractBuffer::readUShort, AbstractBuffer::writeUShort, 42)
val UShort: IMarshaller<UShort> = create(kotlin.UShort::class, AbstractBuffer::readUShort, AbstractBuffer::writeUShort, 42)
@ExperimentalUnsignedTypes
val UInt: IMarshaller<UInt> = create(AbstractBuffer::readUInt, AbstractBuffer::writeUInt, 43)
val UInt: IMarshaller<UInt> = create(kotlin.UInt::class, AbstractBuffer::readUInt, AbstractBuffer::writeUInt, 43)
@ExperimentalUnsignedTypes
val ULong: IMarshaller<ULong> = create(AbstractBuffer::readULong, AbstractBuffer::writeULong, 44)
val ULong: IMarshaller<ULong> = create(kotlin.ULong::class, AbstractBuffer::readULong, AbstractBuffer::writeULong, 44)

@ExperimentalUnsignedTypes
val UByteArray: UniversalMarshaller<UByteArray> = create(AbstractBuffer::readUByteArray, AbstractBuffer::writeUByteArray, 45)
val UByteArray: UniversalMarshaller<UByteArray> = create(kotlin.UByteArray::class, AbstractBuffer::readUByteArray, AbstractBuffer::writeUByteArray, 45)
@ExperimentalUnsignedTypes
val UShortArray: UniversalMarshaller<UShortArray> = create(AbstractBuffer::readUShortArray, AbstractBuffer::writeUShortArray, 46)
val UShortArray: UniversalMarshaller<UShortArray> = create(kotlin.UShortArray::class, AbstractBuffer::readUShortArray, AbstractBuffer::writeUShortArray, 46)
@ExperimentalUnsignedTypes
val UIntArray: UniversalMarshaller<UIntArray> = create(AbstractBuffer::readUIntArray, AbstractBuffer::writeUIntArray, 47)
val UIntArray: UniversalMarshaller<UIntArray> = create(kotlin.UIntArray::class, AbstractBuffer::readUIntArray, AbstractBuffer::writeUIntArray, 47)
@ExperimentalUnsignedTypes
val ULongArray: UniversalMarshaller<ULongArray> = create(AbstractBuffer::readULongArray, AbstractBuffer::writeULongArray, 48)
val ULongArray: UniversalMarshaller<ULongArray> = create(kotlin.ULongArray::class, AbstractBuffer::readULongArray, AbstractBuffer::writeULongArray, 48)

fun registerIn(serializers: ISerializers) {
serializers.register(Int8)
Expand Down
13 changes: 9 additions & 4 deletions rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/Identities.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package com.jetbrains.rd.framework


import com.jetbrains.rd.util.AtomicInteger
import com.jetbrains.rd.util.hash.getPlatformIndependentHash
import com.jetbrains.rd.util.string.condstr

enum class IdKind {
Expand All @@ -10,15 +11,19 @@ enum class IdKind {
}

//PLEASE DO NOT CHANGE IT!!! IT'S EXACTLY THE SAME ON C# SIDE
fun String?.getPlatformIndependentHash(initial: Long = 19L) : Long = this?.fold(initial) { acc, c -> acc*31 + c.toInt()} ?:0
fun Int.getPlatformIndependentHash(initial: Long = 19L) : Long = initial*31 + (this + 1)
fun Long.getPlatformIndependentHash(initial: Long = 19L) : Long = initial*31 + (this + 1)
@Deprecated("Api moved to com.jetbrains.rd.util.hash", ReplaceWith("getPlatformIndependentHash(initial)","com.jetbrains.rd.util.hash.getPlatformIndependentHash"))
fun String?.getPlatformIndependentHash(initial: Long = 19L) : Long = getPlatformIndependentHash(initial)
@Deprecated("Api moved to com.jetbrains.rd.util.hash", ReplaceWith("getPlatformIndependentHash(initial)","com.jetbrains.rd.util.hash.getPlatformIndependentHash"))
fun Int.getPlatformIndependentHash(initial: Long = 19L) : Long = getPlatformIndependentHash(initial)
@Deprecated("Api moved to com.jetbrains.rd.util.hash", ReplaceWith("getPlatformIndependentHash(initial)","com.jetbrains.rd.util.hash.getPlatformIndependentHash"))
fun Long.getPlatformIndependentHash(initial: Long = 19L) : Long = getPlatformIndependentHash(initial)


/**
* An identifier of the object that participates in the object graph.
*/
data class RdId(val hash: Long) {
@JvmInline
value class RdId(val hash: Long) {

companion object {
val Null : RdId = RdId( 0)
Expand Down
28 changes: 27 additions & 1 deletion rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/Interfaces.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ import com.jetbrains.rd.framework.impl.ProtocolContexts
import com.jetbrains.rd.util.lifetime.Lifetime
import com.jetbrains.rd.util.reactive.*
import com.jetbrains.rd.util.string.RName
import java.rmi.NotBoundException
import kotlin.reflect.KClass
import kotlin.reflect.jvm.jvmName

/**
* A node in a graph of entities that can be synchronized with its remote copy over a network or a similar connection.
Expand Down Expand Up @@ -97,6 +97,32 @@ interface IMarshaller<T : Any> : ISerializer<T> {
get() = RdId(_type.simpleName.getPlatformIndependentHash())
}

val IMarshaller<*>.fqn: String get() {
return if (this is LazyCompanionMarshaller) this.fgn
else _type.qualifiedName ?: _type.jvmName
}

class LazyCompanionMarshaller<T : Any>(
override val id: RdId,
val classLoader: ClassLoader,
val fqn: String
) : IMarshaller<T> {
private val lazy = lazy(LazyThreadSafetyMode.PUBLICATION) {
Class.forName(fgn, true, classLoader).getDeclaredField("Companion").get(null) as IMarshaller<T>
}

override val _type: KClass<*>
get() = lazy.value._type

override fun read(ctx: SerializationCtx, buffer: AbstractBuffer): T {
return lazy.value.read(ctx, buffer)
}

override fun write(ctx: SerializationCtx, buffer: AbstractBuffer, value: T) {
lazy.value.write(ctx, buffer, value)
}
}

interface IAbstractDeclaration<T> {
fun readUnknownInstance(ctx: SerializationCtx, buffer: AbstractBuffer, unknownId: RdId, size: Int): T
}
Expand Down
6 changes: 3 additions & 3 deletions rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/MarshallerUtils.kt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.jetbrains.rd.framework

import com.jetbrains.rd.framework.FrameworkMarshallers.create
import com.jetbrains.rd.framework.base.IRdWireable
import com.jetbrains.rd.framework.impl.RdSignal
import com.jetbrains.rd.util.string.RName
Expand All @@ -9,7 +10,7 @@ object RNameMarshaller {
val isEmpty = buffer.readBoolean()
if (isEmpty)
return RName.Empty

val rootName = buffer.readString()
var last = buffer.readBoolean()
var rName = RName(rootName)
Expand Down Expand Up @@ -41,8 +42,7 @@ object RNameMarshaller {
}

internal fun IRdDynamic.createExtSignal(): RdSignal<ExtCreationInfo> {
val marshaller = FrameworkMarshallers.create(
{ buffer ->
val marshaller = create(ExtCreationInfo::class, { buffer ->
val rName = RNameMarshaller.read(buffer)
val rdId = buffer.readNullable { buffer.readRdId() }
val hash = buffer.readLong()
Expand Down
46 changes: 28 additions & 18 deletions rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/Serializers.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@ package com.jetbrains.rd.framework
import com.jetbrains.rd.framework.base.ISerializersOwner
import com.jetbrains.rd.framework.impl.RdSecureString
import com.jetbrains.rd.util.*
import com.jetbrains.rd.util.hash.getPlatformIndependentHash
import com.jetbrains.rd.util.lifetime.Lifetime
import java.util.concurrent.TimeUnit
import kotlin.reflect.KClass
import kotlin.reflect.jvm.jvmName
import kotlin.time.Duration
import kotlin.time.DurationUnit
import kotlin.time.toDuration
Expand Down Expand Up @@ -33,9 +36,8 @@ class Serializers : ISerializers {



val types = hashMapOf<RdId, KClass<*>>()
val writers = hashMapOf<KClass<*>, Pair<RdId, (SerializationCtx, AbstractBuffer, Any) -> Unit>>()
val marshallers = hashMapOf<RdId, IMarshaller<*>>()
private val writers = ConcurrentHashMap<KClass<*>, IMarshaller<*>>()
private val marshallers = ConcurrentHashMap<RdId, IMarshaller<*>>()

init {
backgroundRegistrar.invokeOrQueue {
Expand All @@ -50,19 +52,15 @@ class Serializers : ISerializers {
}

val id = serializer.id
val t = serializer._type

Protocol.initializationLogger.trace { "Registering type ${t.simpleName}, id = $id" }

val existing = types[id]
val existing = marshallers[id]
if (existing != null) {
require(existing == t) { "Can't register ${t.simpleName} with id: $id, already registered: ${existing.simpleName}" }
} else {
types[id] = t
require(existing.fqn == serializer.fqn) { "Can't register ${serializer.fqn} with id: $id, already registered: ${serializer.fqn}" }
} else {
Protocol.initializationLogger.trace { "Registering type ${serializer.fqn}, id = $id" }
marshallers[id] = serializer
if (serializer !is LazyCompanionMarshaller)
writers[serializer._type] = serializer
}

marshallers[id] = serializer
writers[t] = Pair(id, serializer::write) as Pair<RdId, (SerializationCtx, AbstractBuffer, Any) -> Unit>
}

override fun get(id: RdId): IMarshaller<*>? {
Expand Down Expand Up @@ -102,22 +100,34 @@ class Serializers : ISerializers {
?: throw IllegalStateException("Non-null object expected")
}

private fun <T : Any> getWriter(clazz: KClass<out T>): IMarshaller<T> {
val marshaller = writers.getOrPut(clazz) {
val id = RdId(clazz.simpleName.getPlatformIndependentHash())
marshallers[id] ?: cantFindWriter(clazz)
}

return marshaller as? IMarshaller<T> ?: cantFindWriter(clazz)
}

private fun <T : Any> cantFindWriter(clazz: KClass<T>): Nothing {
throw IllegalStateException("Can't find writer by class: ${clazz}. $notRegisteredErrorMessage")
}

override fun <T : Any> writePolymorphic(ctx: SerializationCtx, stream: AbstractBuffer, value: T) {
backgroundRegistrar.flush()

val (id, writer) = writers[value::class]
?: throw IllegalStateException("Can't find writer by class: ${value::class}. $notRegisteredErrorMessage")
val serializer = getWriter(value::class)

if (value is IUnknownInstance) {
value.unknownId.write(stream)
} else {
id.write(stream)
serializer.id.write(stream)
}

val lengthTagPosition = stream.position
stream.writeInt(0)
val objectStartPosition = stream.position
writer(ctx, stream, value)
serializer.write(ctx, stream, value)
val objectEndPosition = stream.position
stream.position = lengthTagPosition
stream.writeInt(objectEndPosition - objectStartPosition)
Expand Down
2 changes: 1 addition & 1 deletion rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/RdGen.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ class RdGen : Kli() {
/**
* Moving this field forward you trigger rebuild even if inputs and output of generator hasn't changed.
*/
const val version = "1.12"
const val version = "1.13"

/**
* File to store all information for incremental work
Expand Down
Loading

0 comments on commit 63d16a6

Please sign in to comment.