diff --git a/lib/IRGen/TypeLayout.cpp b/lib/IRGen/TypeLayout.cpp index 5dbb10abbfd5f..c89ea702061fe 100644 --- a/lib/IRGen/TypeLayout.cpp +++ b/lib/IRGen/TypeLayout.cpp @@ -77,7 +77,12 @@ class LayoutStringBuilder { Resilient = 0x0f, SinglePayloadEnumSimple = 0x10, SinglePayloadEnumFN = 0x11, - SinglePayloadEnumFNResolved = 0x12, + // reserved + // SinglePayloadEnumFNResolved = 0x12, + + MultiPayloadEnumFN = 0x13, + // reserved + // MultiPayloadEnumFNResolved = 0x14, Skip = 0x80, // We may use the MSB as flag that a count follows, @@ -101,13 +106,19 @@ class LayoutStringBuilder { const TypeLayoutEntry *payload; }; + struct MultiPayloadEnumFN { + llvm::Function *tagFn; + const EnumTypeLayoutEntry *entry; + }; + struct RefCounting { RefCountingKind kind; union { size_t size; - llvm::Function* metaTypeRef; + llvm::Function *metaTypeRef; SinglePayloadEnumSimple singlePayloadEnumSimple; SinglePayloadEnumFN singlePayloadEnumFN; + MultiPayloadEnumFN multiPayloadEnumFN; }; RefCounting() = default; @@ -151,6 +162,15 @@ class LayoutStringBuilder { refCountings.push_back(op); } + void addMultiPayloadEnumFN(llvm::Function *tagFn, + const EnumTypeLayoutEntry *entry) { + RefCounting op; + op.kind = RefCountingKind::MultiPayloadEnumFN; + op.multiPayloadEnumFN.tagFn = tagFn; + op.multiPayloadEnumFN.entry = entry; + refCountings.push_back(op); + } + void addSkip(size_t size) { if (refCountings.empty() || refCountings.back().kind != RefCountingKind::Skip) { @@ -280,6 +300,63 @@ class LayoutStringBuilder { break; } + case RefCountingKind::MultiPayloadEnumFN: { + uint64_t op = (static_cast(refCounting.kind) << 56) | skip; + B.addInt64(op); + + skip = 0; + + auto enumData = refCounting.multiPayloadEnumFN; + auto payloads = enumData.entry->cases; + + B.addRelativeOffset(IGM.IntPtrTy, enumData.tagFn); + + B.addSize(Size(payloads.size())); + + auto nestedRefCountBytesPlaceholder = + B.addPlaceholderWithSize(IGM.SizeTy); + B.addSize(*enumData.entry->fixedSize(IGM)); + + SmallVector< + clang::CodeGen::ConstantAggregateBuilderBase::PlaceholderPosition, + 4> + offsetPlaceholders; + for (auto *p : payloads) { + (void)p; + auto placeholder = B.addPlaceholderWithSize(IGM.SizeTy); + offsetPlaceholders.push_back(placeholder); + refCountBytes += IGM.getPointerSize().getValue(); + } + + size_t nestedRefCountBytes = 0; + for (auto p : llvm::zip(payloads, offsetPlaceholders)) { + auto *payload = std::get<0>(p); + + B.fillPlaceholderWithInt(std::get<1>(p), IGM.SizeTy, + nestedRefCountBytes); + + size_t nestedSkip = 0; + LayoutStringBuilder nestedBuilder{}; + payload->refCountString(IGM, nestedBuilder, genericSig); + addRefCountings(IGM, B, genericSig, nestedBuilder.refCountings, + nestedSkip, nestedRefCountBytes, flags); + + // NUL terminator + B.addInt64(0); + nestedRefCountBytes += sizeof(uint64_t); + } + + B.fillPlaceholderWithInt(nestedRefCountBytesPlaceholder, IGM.SizeTy, + nestedRefCountBytes); + + refCountBytes += sizeof(uint64_t) + + (4 * IGM.getPointerSize().getValue()) + + nestedRefCountBytes; + + flags |= LayoutStringFlags::HasRelativePointers; + break; + } + default: { uint64_t op = (static_cast(refCounting.kind) << 56) | skip; B.addInt64(op); @@ -2208,6 +2285,22 @@ bool EnumTypeLayoutEntry::buildSinglePayloadRefCountString( return true; } +bool EnumTypeLayoutEntry::buildMultiPayloadRefCountString( + IRGenModule &IGM, LayoutStringBuilder &B, + GenericSignature genericSig) const { + auto valid = std::all_of(cases.begin(), cases.end(), [&](auto *c) { + LayoutStringBuilder nestedBuilder{}; + return c->refCountString(IGM, nestedBuilder, genericSig); + }); + + if (valid) { + auto *tagFn = createFixedEnumLoadTag(IGM, *this); + B.addMultiPayloadEnumFN(tagFn, this); + } + + return valid; +} + llvm::Constant * EnumTypeLayoutEntry::layoutString(IRGenModule &IGM, GenericSignature genericSig) const { @@ -2245,12 +2338,20 @@ EnumTypeLayoutEntry::layoutString(IRGenModule &IGM, return nullptr; case CopyDestroyStrategy::Normal: { - if (!isFixedSize(IGM) || isMultiPayloadEnum() || - !buildSinglePayloadRefCountString(IGM, B, genericSig)) { - return *(_layoutString = llvm::Optional(nullptr)); + bool valid = false; + if (isFixedSize(IGM)) { + if (isMultiPayloadEnum()) { + valid = buildMultiPayloadRefCountString(IGM, B, genericSig); + } else { + valid = buildSinglePayloadRefCountString(IGM, B, genericSig); + } } - return createConstant(B); + if (valid) { + return createConstant(B); + } else { + return *(_layoutString = llvm::Optional(nullptr)); + } } case CopyDestroyStrategy::ForwardToPayload: @@ -2281,8 +2382,11 @@ bool EnumTypeLayoutEntry::refCountString(IRGenModule &IGM, case CopyDestroyStrategy::ForwardToPayload: return cases[0]->refCountString(IGM, B, genericSig); case CopyDestroyStrategy::Normal: { - if (!isMultiPayloadEnum() && - buildSinglePayloadRefCountString(IGM, B, genericSig)) { + + if (isMultiPayloadEnum() && + buildMultiPayloadRefCountString(IGM, B, genericSig)) { + return true; + } else if (buildSinglePayloadRefCountString(IGM, B, genericSig)) { return true; } diff --git a/lib/IRGen/TypeLayout.h b/lib/IRGen/TypeLayout.h index 9252caa1f47e9..dc2b13eda35e2 100644 --- a/lib/IRGen/TypeLayout.h +++ b/lib/IRGen/TypeLayout.h @@ -651,6 +651,9 @@ class EnumTypeLayoutEntry : public TypeLayoutEntry, LayoutStringBuilder &B, GenericSignature genericSig) const; + bool buildMultiPayloadRefCountString(IRGenModule &IGM, LayoutStringBuilder &B, + GenericSignature genericSig) const; + static bool classof(const TypeLayoutEntry *entry); }; diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 9f72acd42f93c..0917e98331e58 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -109,8 +109,10 @@ static void existential_destroy(OpaqueValue* object) { } } -template -inline static bool handleNextRefCount(const Metadata *metadata, const uint8_t *typeLayout, size_t &offset, uintptr_t &addrOffset, Params... params) { +template +inline static bool handleNextRefCount(const Metadata *metadata, + const uint8_t *typeLayout, size_t &offset, + uintptr_t &addrOffset, Params... params) { uint64_t skip = readBytes(typeLayout, offset); auto tag = static_cast(skip >> 56); skip &= ~(0xffULL << 56); @@ -126,7 +128,8 @@ inline static bool handleNextRefCount(const Metadata *metadata, const uint8_t *t Handler::handleMetatype(type, addrOffset, std::forward(params)...); } else if (SWIFT_UNLIKELY(tag == RefCountingKind::SinglePayloadEnumSimple)) { - Handler::handleSinglePayloadEnumSimple(typeLayout, offset, addrOffset, std::forward(params)...); + Handler::handleSinglePayloadEnumSimple(typeLayout, offset, addrOffset, + std::forward(params)...); } else if (SWIFT_UNLIKELY(tag == RefCountingKind::SinglePayloadEnumFN)) { Handler::handleSinglePayloadEnumFN(typeLayout, offset, false, addrOffset, std::forward(params)...); @@ -134,6 +137,15 @@ inline static bool handleNextRefCount(const Metadata *metadata, const uint8_t *t RefCountingKind::SinglePayloadEnumFNResolved)) { Handler::handleSinglePayloadEnumFN(typeLayout, offset, true, addrOffset, std::forward(params)...); + } else if (SWIFT_UNLIKELY(tag == RefCountingKind::MultiPayloadEnumFN)) { + Handler::handleMultiPayloadEnumFN(metadata, typeLayout, offset, false, + addrOffset, + std::forward(params)...); + } else if (SWIFT_UNLIKELY(tag == + RefCountingKind::MultiPayloadEnumFNResolved)) { + Handler::handleMultiPayloadEnumFN(metadata, typeLayout, offset, true, + addrOffset, + std::forward(params)...); } else { Handler::handleReference(tag, addrOffset, std::forward(params)...); } @@ -141,21 +153,31 @@ inline static bool handleNextRefCount(const Metadata *metadata, const uint8_t *t return true; } -template -inline static void handleRefCounts(const Metadata *metadata, Params... params) { - const uint8_t *typeLayout = metadata->getLayoutString(); - size_t offset = layoutStringHeaderSize; - uintptr_t addrOffset = 0; - +template +inline static void handleRefCounts(const Metadata *metadata, + const uint8_t *typeLayout, size_t &offset, + uintptr_t &addrOffset, Params... params) { if (N == 0) { - while (handleNextRefCount(metadata, typeLayout, offset, addrOffset, std::forward(params)...)) {} + while (handleNextRefCount(metadata, typeLayout, offset, addrOffset, + std::forward(params)...)) { + } } else { for (int i = 0; i < N; i++) { - handleNextRefCount(metadata, typeLayout, offset, addrOffset, std::forward(params)...); + handleNextRefCount(metadata, typeLayout, offset, addrOffset, + std::forward(params)...); } } } +template +inline static void handleRefCounts(const Metadata *metadata, Params... params) { + const uint8_t *typeLayout = metadata->getLayoutString(); + size_t offset = layoutStringHeaderSize; + uintptr_t addrOffset = 0; + handleRefCounts(metadata, typeLayout, offset, addrOffset, + std::forward(params)...); +} + static uint64_t readTagBytes(uint8_t *addr, uint8_t byteCount) { switch (byteCount) { case 1: @@ -173,7 +195,7 @@ static uint64_t readTagBytes(uint8_t *addr, uint8_t byteCount) { static void handleSinglePayloadEnumSimple(const uint8_t *typeLayout, size_t &offset, uint8_t *addr, - size_t &addrOffset) { + uintptr_t &addrOffset) { auto byteCountsAndOffset = readBytes(typeLayout, offset); auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; @@ -220,7 +242,7 @@ typedef unsigned (*GetEnumTagFn)(const uint8_t *); static void handleSinglePayloadEnumFN(const uint8_t *typeLayout, size_t &offset, bool resolved, uint8_t *addr, - size_t &addrOffset) { + uintptr_t &addrOffset) { GetEnumTagFn getEnumTag; if (resolved) { getEnumTag = readBytes(typeLayout, offset); @@ -240,6 +262,39 @@ static void handleSinglePayloadEnumFN(const uint8_t *typeLayout, size_t &offset, } } +template +static void handleMultiPayloadEnumFN(const Metadata *metadata, + const uint8_t *typeLayout, size_t &offset, + bool resolved, uintptr_t &addrOffset, + uint8_t *addr, Params... params) { + GetEnumTagFn getEnumTag; + if (resolved) { + getEnumTag = readBytes(typeLayout, offset); + } else { + getEnumTag = readRelativeFunctionPointer(typeLayout, offset); + } + + size_t numCases = readBytes(typeLayout, offset); + size_t refCountBytes = readBytes(typeLayout, offset); + size_t enumSize = readBytes(typeLayout, offset); + + unsigned enumTag = getEnumTag(addr + addrOffset); + + if (enumTag < numCases) { + size_t nestedOffset = offset + (enumTag * sizeof(size_t)); + size_t refCountOffset = readBytes(typeLayout, nestedOffset); + nestedOffset = offset + (numCases * sizeof(size_t)) + refCountOffset; + + uintptr_t nestedAddrOffset = addrOffset; + handleRefCounts<0, Handler>(metadata, typeLayout, nestedOffset, + nestedAddrOffset, addr, + std::forward(params)...); + } + + offset += refCountBytes + (numCases * sizeof(size_t)); + addrOffset += enumSize; +} + const DestroyFuncAndMask destroyTable[] = { {(DestrFn)&skipDestroy, false}, {(DestrFn)&swift_errorRelease, true}, @@ -265,23 +320,36 @@ const DestroyFuncAndMask destroyTable[] = { }; struct DestroyHandler { - static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, uint8_t *addr) { + static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, + uint8_t *addr) { type->vw_destroy((OpaqueValue *)(addr + addrOffset)); } - static inline void handleSinglePayloadEnumSimple(const uint8_t *typeLayout, size_t &offset, - size_t &addrOffset, uint8_t *addr) { + static inline void handleSinglePayloadEnumSimple(const uint8_t *typeLayout, + size_t &offset, + uintptr_t &addrOffset, + uint8_t *addr) { ::handleSinglePayloadEnumSimple(typeLayout, offset, addr, addrOffset); } static inline void handleSinglePayloadEnumFN(const uint8_t *typeLayout, size_t &offset, bool resolved, - size_t &addrOffset, + uintptr_t &addrOffset, uint8_t *addr) { ::handleSinglePayloadEnumFN(typeLayout, offset, resolved, addr, addrOffset); } - static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset, uint8_t *addr) { + static inline void handleMultiPayloadEnumFN(const Metadata *metadata, + const uint8_t *typeLayout, + size_t &offset, bool resolved, + uintptr_t &addrOffset, + uint8_t *addr) { + ::handleMultiPayloadEnumFN(metadata, typeLayout, offset, + resolved, addrOffset, addr); + } + + static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset, + uint8_t *addr) { const auto &destroyFunc = destroyTable[static_cast(tag)]; if (SWIFT_LIKELY(destroyFunc.isIndirect)) { destroyFunc.fn( @@ -344,25 +412,38 @@ const RetainFuncAndMask retainTable[] = { }; struct CopyHandler { - static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, uint8_t *dest, uint8_t *src) { + static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, + uint8_t *dest, uint8_t *src) { type->vw_initializeWithCopy((OpaqueValue*)((uintptr_t)dest + addrOffset), (OpaqueValue*)((uintptr_t)src + addrOffset)); } - static inline void handleSinglePayloadEnumSimple(const uint8_t *typeLayout, size_t &offset, - size_t &addrOffset, uint8_t *dest, uint8_t *src) { - ::handleSinglePayloadEnumSimple(typeLayout, offset, (uint8_t *)src, addrOffset); + static inline void handleSinglePayloadEnumSimple(const uint8_t *typeLayout, + size_t &offset, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + ::handleSinglePayloadEnumSimple(typeLayout, offset, src, addrOffset); } static inline void handleSinglePayloadEnumFN(const uint8_t *typeLayout, size_t &offset, bool resolved, - size_t &addrOffset, + uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - ::handleSinglePayloadEnumFN(typeLayout, offset, resolved, (uint8_t *)src, - addrOffset); + ::handleSinglePayloadEnumFN(typeLayout, offset, resolved, src, addrOffset); + } + + static inline void handleMultiPayloadEnumFN(const Metadata *metadata, + const uint8_t *typeLayout, + size_t &offset, bool resolved, + uintptr_t &addrOffset, + uint8_t *dest, uint8_t *src) { + ::handleMultiPayloadEnumFN(metadata, typeLayout, offset, + resolved, addrOffset, dest, src); } - static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset, uint8_t *dest, uint8_t *src) { + static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset, + uint8_t *dest, uint8_t *src) { const auto &retainFunc = retainTable[static_cast(tag)]; if (SWIFT_LIKELY(retainFunc.isSingle)) { ((RetainFn)retainFunc.fn)(*(void**)(((uintptr_t)dest + addrOffset))); @@ -409,6 +490,15 @@ struct TakeHandler { ::handleSinglePayloadEnumFN(typeLayout, offset, resolved, src, addrOffset); } + static inline void handleMultiPayloadEnumFN(const Metadata *metadata, + const uint8_t *typeLayout, + size_t &offset, bool resolved, + uintptr_t &addrOffset, + uint8_t *dest, uint8_t *src) { + ::handleMultiPayloadEnumFN(metadata, typeLayout, offset, + resolved, addrOffset, dest, src); + } + static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset, uint8_t *dest, uint8_t *src) { if (tag == RefCountingKind::UnknownWeak) { @@ -420,8 +510,8 @@ struct TakeHandler { (OpaqueValue*)((uintptr_t)src + addrOffset)); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { type->vw_initializeWithTake( - (OpaqueValue*)((uintptr_t)dest + addrOffset), - (OpaqueValue*)((uintptr_t)src + addrOffset)); + (OpaqueValue *)((uintptr_t)dest + addrOffset), + (OpaqueValue *)((uintptr_t)src + addrOffset)); } } } @@ -457,17 +547,20 @@ swift_generic_assignWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src, return swift_generic_initWithTake(dest, src, metadata); } -void swift::swift_resolve_resilientAccessors( - uint8_t *layoutStr, size_t layoutStrOffset, const uint8_t *fieldLayoutStr, - size_t refCountBytes, const Metadata *fieldType) { +void swift::swift_resolve_resilientAccessors(uint8_t *layoutStr, + size_t layoutStrOffset, + const uint8_t *fieldLayoutStr, + const Metadata *fieldType) { size_t i = layoutStringHeaderSize; - while (i < (layoutStringHeaderSize + refCountBytes)) { + while (true) { size_t currentOffset = i; uint64_t size = readBytes(fieldLayoutStr, i); RefCountingKind tag = (RefCountingKind)(size >> 56); size &= ~(0xffULL << 56); switch (tag) { + case RefCountingKind::End: + return; case RefCountingKind::Resilient: { auto *type = getResilientTypeMetadata(fieldType, fieldLayoutStr, i); size_t writeOffset = layoutStrOffset + currentOffset - @@ -503,6 +596,44 @@ void swift::swift_resolve_resilientAccessors( i += 3 * sizeof(size_t); break; + case RefCountingKind::MultiPayloadEnumFN: { + auto getEnumTag = + readRelativeFunctionPointer(fieldLayoutStr, i); + size_t writeOffset = + layoutStrOffset + currentOffset - layoutStringHeaderSize; + uint64_t tagAndOffset = + (((uint64_t)RefCountingKind::MultiPayloadEnumFNResolved) << 56) | + size; + writeBytes(layoutStr, writeOffset, tagAndOffset); + writeBytes(layoutStr, writeOffset, getEnumTag); + + size_t numCases = readBytes(fieldLayoutStr, i); + size_t refCountBytes = readBytes(fieldLayoutStr, i); + + size_t casesBeginOffset = + layoutStrOffset + i + (numCases * sizeof(size_t)); + + for (size_t j = 0; j < numCases; j++) { + size_t caseOffset = readBytes(fieldLayoutStr, i); + const uint8_t *caseLayoutString = + fieldLayoutStr + i + (numCases * sizeof(size_t)) + caseOffset; + swift_resolve_resilientAccessors(layoutStr, + casesBeginOffset + caseOffset, + caseLayoutString, fieldType); + } + break; + } + + case RefCountingKind::MultiPayloadEnumFNResolved: { + // skip function pointer + i += sizeof(uintptr_t); + size_t numCases = readBytes(fieldLayoutStr, i); + size_t refCountBytes = readBytes(fieldLayoutStr, i); + // skip enum size, offsets and ref counts + i += sizeof(size_t) + (numCases * sizeof(size_t)) + refCountBytes; + break; + } + default: break; } diff --git a/stdlib/public/runtime/BytecodeLayouts.h b/stdlib/public/runtime/BytecodeLayouts.h index 02bf2e06ceb38..f589352c25a17 100644 --- a/stdlib/public/runtime/BytecodeLayouts.h +++ b/stdlib/public/runtime/BytecodeLayouts.h @@ -47,6 +47,9 @@ enum class RefCountingKind : uint8_t { SinglePayloadEnumFN = 0x11, SinglePayloadEnumFNResolved = 0x12, + MultiPayloadEnumFN = 0x13, + MultiPayloadEnumFNResolved = 0x14, + Skip = 0x80, // We may use the MSB as flag that a count follows, // so all following values are reserved @@ -66,7 +69,10 @@ swift::OpaqueValue *swift_generic_initWithTake(swift::OpaqueValue *dest, swift:: SWIFT_RUNTIME_EXPORT void swift_generic_instantiateLayoutString(const uint8_t *layoutStr, Metadata *type); -void swift_resolve_resilientAccessors(uint8_t *layoutStr, size_t layoutStrOffset, const uint8_t *fieldLayoutStr, size_t refCountBytes, const Metadata *fieldType); +void swift_resolve_resilientAccessors(uint8_t *layoutStr, + size_t layoutStrOffset, + const uint8_t *fieldLayoutStr, + const Metadata *fieldType); } // namespace swift #endif // SWIFT_BYTECODE_LAYOUTS_H diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index f891026e580e8..cae6faac16b74 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2803,8 +2803,7 @@ void swift::swift_initStructMetadataWithLayoutString( if (fieldFlags & LayoutStringFlags::HasRelativePointers) { swift_resolve_resilientAccessors(layoutStr, layoutStrOffset, - fieldLayoutStr, fieldRefCountBytes, - fieldType); + fieldLayoutStr, fieldType); } if (offset) {