1//===- SemanticSignaturePacking.cpp - HLSL signature packing helpers -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file This file implements helpers for packing HLSL semantic signatures.
10///
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Frontend/HLSL/SemanticSignaturePacking.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/Sequence.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/ADT/bit.h"
18#include <algorithm>
19#include <array>
20#include <cassert>
21#include <cstdint>
22#include <limits>
23#include <optional>
24
25using namespace llvm;
26using namespace llvm::hlsl;
27
28char SignaturePackingError::ID;
29
30namespace {
31
32// The range of rows covered by a dynamically indexable element. Only an
33// element that covers multiple rows is dynamically indexable, so a single-row
34// element has an empty range.
35struct IndexedRowRange {
36 uint8_t Begin = 0;
37 uint8_t End = 0;
38
39 static IndexedRowRange of(unsigned StartRow, unsigned RowCount) {
40 if (RowCount < 2)
41 return {};
42 return {.Begin: static_cast<uint8_t>(StartRow),
43 .End: static_cast<uint8_t>(StartRow + RowCount)};
44 }
45
46 bool isEmpty() const { return Begin == End; }
47
48 // An empty range is contained by every range.
49 bool contains(IndexedRowRange Other) const {
50 return Other.isEmpty() || (Begin <= Other.Begin && Other.End <= End);
51 }
52
53 IndexedRowRange unionWith(IndexedRowRange Other) const {
54 if (isEmpty())
55 return Other;
56 if (Other.isEmpty())
57 return *this;
58 return {.Begin: std::min(a: Begin, b: Other.Begin), .End: std::max(a: End, b: Other.End)};
59 }
60
61 bool operator==(IndexedRowRange Other) const {
62 return Begin == Other.Begin && End == Other.End;
63 }
64};
65
66static_assert(SemanticInterpretation::Arbitrary < SemanticInterpretation::SV &&
67 SemanticInterpretation::SV < SemanticInterpretation::SGV &&
68 SemanticInterpretation::SGV <
69 SemanticInterpretation::ClipCull &&
70 SemanticInterpretation::ClipCull <
71 SemanticInterpretation::TessFactor,
72 "semantic interpretations must be in component packing order");
73
74struct SignatureRow {
75 uint8_t OccupiedColumns = 0;
76 IndexedRowRange IndexedRange;
77 bool IndexedRangeFixed = false;
78 unsigned ComponentWidth = 0;
79 dxbc::PSV::InterpolationMode InterpMode =
80 dxbc::PSV::InterpolationMode::Undefined;
81 SemanticInterpretation RightmostInterpretation =
82 SemanticInterpretation::Arbitrary;
83};
84
85using SignatureRows = std::array<SignatureRow, MaxSignatureRows>;
86
87struct ElementPlacement {
88 unsigned Rows;
89 unsigned Cols;
90 unsigned ComponentWidth;
91 dxbc::PSV::InterpolationMode InterpMode;
92 SemanticInterpretation Interpretation;
93};
94
95struct ElementLocation {
96 uint32_t Row = UnallocatedRow;
97 uint8_t Col = UnallocatedCol;
98};
99
100struct OptimizedClipCullElement {
101 unsigned Index;
102 ElementPlacement Placement;
103 ElementLocation Location;
104};
105
106// Clip/cull elements are first packed into an independent two-row grid. Each
107// row used in that grid maps to a whole reserved row in the signature.
108struct ClipCullState {
109 std::array<SignatureRow, MaxClipCullRows> Rows;
110 std::array<unsigned, MaxClipCullRows> SignatureRows = {UnallocatedRow,
111 UnallocatedRow};
112 unsigned RowsUsed = 0;
113};
114
115enum class PackingGroup : unsigned {
116 FullRegister,
117 IndexedTessFactor,
118 Arbitrary,
119 SystemValue,
120 ClipCull,
121 SystemGenerated,
122 NotAllocated,
123};
124
125} // namespace
126
127static uint8_t getStartColumn(uint8_t ColumnMask) {
128 assert(ColumnMask != 0 && "expected at least one occupied column");
129 return countr_zero(Val: ColumnMask);
130}
131
132static PackingGroup
133getOptimizedPackingGroup(const SemanticSignatureElement &Element,
134 Triple::EnvironmentType ShaderStage, IOType IOTy) {
135 const SemanticInterpretation Interpretation =
136 getInterpretationKind(SemanticKind: Element.SemanticKind, ShaderStage, IOTy);
137 assert((Interpretation != SemanticInterpretation::Invalid &&
138 Interpretation != SemanticInterpretation::Target) &&
139 "unexpected semantic interpretation for optimized packing, "
140 "should have been diagnosed by Sema");
141
142 if (Element.Cols == MaxSignatureCols &&
143 (Interpretation == SemanticInterpretation::Arbitrary ||
144 Interpretation == SemanticInterpretation::SV))
145 return PackingGroup::FullRegister;
146
147 if (Interpretation == SemanticInterpretation::TessFactor && Element.Rows > 1)
148 return PackingGroup::IndexedTessFactor;
149
150 switch (Interpretation) {
151 case SemanticInterpretation::Arbitrary:
152 return PackingGroup::Arbitrary;
153 case SemanticInterpretation::SV:
154 case SemanticInterpretation::TessFactor:
155 return PackingGroup::SystemValue;
156 case SemanticInterpretation::ClipCull:
157 return PackingGroup::ClipCull;
158 case SemanticInterpretation::SGV:
159 return PackingGroup::SystemGenerated;
160 case SemanticInterpretation::NotAllocated:
161 return PackingGroup::NotAllocated;
162 case SemanticInterpretation::Invalid:
163 case SemanticInterpretation::Target:
164 break;
165 }
166 llvm_unreachable("unexpected semantic interpretation for optimized packing");
167}
168
169static unsigned getComponentWidth(dxil::ElementType ComponentType,
170 bool UseNative16BitTypes) {
171 assert(ComponentType != dxil::ElementType::I64 &&
172 ComponentType != dxil::ElementType::U64 &&
173 ComponentType != dxil::ElementType::F64 &&
174 ComponentType != dxil::ElementType::SNormF64 &&
175 ComponentType != dxil::ElementType::UNormF64 &&
176 "64-bit types cannot be used in a signature");
177
178 switch (ComponentType) {
179 case dxil::ElementType::F16:
180 case dxil::ElementType::I16:
181 case dxil::ElementType::U16:
182 case dxil::ElementType::SNormF16:
183 case dxil::ElementType::UNormF16:
184 // Without native 16-bit types these are min-precision types that occupy a
185 // whole 32-bit component.
186 return UseNative16BitTypes ? 16 : 32;
187 default:
188 // A boolean is loaded and stored as a 32-bit value.
189 return 32;
190 }
191}
192
193static ElementPlacement
194getElementPlacement(const SemanticSignatureElement &Element,
195 SemanticInterpretation Interpretation,
196 bool UseNative16BitTypes) {
197 // Only indexed tessellation factors need the reserved last column.
198 if (Interpretation == SemanticInterpretation::TessFactor && Element.Rows == 1)
199 Interpretation = SemanticInterpretation::SV;
200 return {.Rows: Element.Rows, .Cols: Element.Cols,
201 .ComponentWidth: getComponentWidth(ComponentType: Element.CompType, UseNative16BitTypes),
202 .InterpMode: Element.InterpMode, .Interpretation: Interpretation};
203}
204
205static SemanticInterpretation
206getComponentOrder(SemanticInterpretation Interpretation) {
207 // Clip/cull values have system-value component ordering, but may be indexed.
208 return Interpretation == SemanticInterpretation::ClipCull
209 ? SemanticInterpretation::SV
210 : Interpretation;
211}
212
213// Returns whether Placement may be co-packed into a Row that it covers, where
214// IndexedRange is the range of rows that it is dynamically indexed over.
215static bool canCoPack(const SignatureRow &Row,
216 const ElementPlacement &Placement,
217 IndexedRowRange IndexedRange) {
218 const bool IsSystemValue =
219 Placement.Interpretation == SemanticInterpretation::SV ||
220 Placement.Interpretation == SemanticInterpretation::SGV;
221
222 // A system value is never dynamically indexable, so it cannot be placed in a
223 // row that is.
224 if (IsSystemValue && !Row.IndexedRange.isEmpty())
225 return false;
226
227 // A row whose indexed range is fixed only accepts elements that are indexed
228 // within that range.
229 if (Row.IndexedRangeFixed && !Row.IndexedRange.contains(Other: IndexedRange))
230 return false;
231
232 // A tess factor fixes the indexed range of the rows it is reserved in, so it
233 // may only extend the range that those rows already have.
234 if (Placement.Interpretation == SemanticInterpretation::TessFactor &&
235 !IndexedRange.contains(Other: Row.IndexedRange))
236 return false;
237
238 if (Row.OccupiedColumns && Row.ComponentWidth != Placement.ComponentWidth)
239 return false;
240 if (Row.InterpMode != dxbc::PSV::InterpolationMode::Undefined &&
241 Row.InterpMode != Placement.InterpMode)
242 return false;
243 // Do not append an earlier semantic category after a later one in a row.
244 // Indexed tess factors are reserved in the last column, so arbitrary values
245 // may still fill the columns to their left without violating that ordering.
246 if (Row.OccupiedColumns &&
247 getComponentOrder(Interpretation: Placement.Interpretation) <
248 getComponentOrder(Interpretation: Row.RightmostInterpretation) &&
249 !(Placement.Interpretation == SemanticInterpretation::Arbitrary &&
250 Row.RightmostInterpretation == SemanticInterpretation::TessFactor))
251 return false;
252 return true;
253}
254
255// Returns the columns that Placement would occupy if it was placed at StartRow,
256// or nullopt if it cannot be placed there.
257static std::optional<uint8_t> canPlaceAt(ArrayRef<SignatureRow> Rows,
258 unsigned StartRow,
259 const ElementPlacement &Placement) {
260 if (StartRow >= Rows.size() || Placement.Rows > Rows.size() - StartRow)
261 return std::nullopt;
262
263 const IndexedRowRange IndexedRange =
264 IndexedRowRange::of(StartRow, RowCount: Placement.Rows);
265 uint8_t OccupiedColumns = 0;
266 for (unsigned ElementRow = 0; ElementRow != Placement.Rows; ++ElementRow) {
267 const SignatureRow &Row = Rows[StartRow + ElementRow];
268 if (!canCoPack(Row, Placement, IndexedRange))
269 return std::nullopt;
270 OccupiedColumns |= Row.OccupiedColumns;
271 }
272
273 // An indexed tess factor is reserved in the last column so that other
274 // elements can still be co-packed into the rows that it covers.
275 if (Placement.Interpretation == SemanticInterpretation::TessFactor) {
276 constexpr uint8_t LastColumn = 1U << (MaxSignatureCols - 1);
277 if (Placement.Cols != 1 || (OccupiedColumns & LastColumn))
278 return std::nullopt;
279 return LastColumn;
280 }
281
282 for (unsigned StartCol = 0; StartCol + Placement.Cols <= MaxSignatureCols;
283 ++StartCol) {
284 const uint8_t ColumnMask = static_cast<uint8_t>(
285 ((1U << Placement.Cols) - 1U) << static_cast<unsigned>(StartCol));
286 if (!(OccupiedColumns & ColumnMask))
287 return ColumnMask;
288 }
289 return std::nullopt;
290}
291
292static void placeRowsAt(MutableArrayRef<SignatureRow> Rows, unsigned StartRow,
293 const ElementPlacement &Placement, uint8_t ColumnMask) {
294 const IndexedRowRange IndexedRange =
295 IndexedRowRange::of(StartRow, RowCount: Placement.Rows);
296 for (unsigned ElementRow = 0; ElementRow != Placement.Rows; ++ElementRow) {
297 SignatureRow &Row = Rows[StartRow + ElementRow];
298 assert(!(Row.OccupiedColumns & ColumnMask) &&
299 "cannot overlap signature elements");
300 const uint8_t PreviousOccupiedColumns = Row.OccupiedColumns;
301 if (!PreviousOccupiedColumns)
302 Row.ComponentWidth = Placement.ComponentWidth;
303 Row.OccupiedColumns |= ColumnMask;
304 if (Row.InterpMode == dxbc::PSV::InterpolationMode::Undefined)
305 Row.InterpMode = Placement.InterpMode;
306 // Non-overlapping masks compare according to their rightmost set bit.
307 if (!PreviousOccupiedColumns || ColumnMask > PreviousOccupiedColumns)
308 Row.RightmostInterpretation = Placement.Interpretation;
309
310 Row.IndexedRange = Row.IndexedRange.unionWith(Other: IndexedRange);
311 if (Placement.Interpretation == SemanticInterpretation::SV ||
312 Placement.Interpretation == SemanticInterpretation::SGV ||
313 Placement.Interpretation == SemanticInterpretation::TessFactor) {
314 assert(Row.IndexedRange == IndexedRange && "incompatible index range");
315 Row.IndexedRangeFixed = true;
316 }
317 }
318}
319
320static void placeAt(MutableArrayRef<SignatureRow> Rows, unsigned StartRow,
321 const ElementPlacement &Placement, uint8_t ColumnMask,
322 ElementLocation &Location) {
323 placeRowsAt(Rows, StartRow, Placement, ColumnMask);
324 Location.Row = StartRow;
325 Location.Col = getStartColumn(ColumnMask);
326}
327
328static bool prefixPackElement(ElementLocation &Location,
329 MutableArrayRef<SignatureRow> Rows,
330 const ElementPlacement &Placement) {
331 for (unsigned StartRow = 0; StartRow != Rows.size(); ++StartRow) {
332 std::optional<uint8_t> ColumnMask = canPlaceAt(Rows, StartRow, Placement);
333 if (!ColumnMask)
334 continue;
335 placeAt(Rows, StartRow, Placement, ColumnMask: *ColumnMask, Location);
336 return true;
337 }
338 return false;
339}
340
341// A clip/cull grid row is backed by a whole reserved signature row.
342static ElementPlacement
343getClipCullReservation(const ElementPlacement &Placement, unsigned RowCount) {
344 ElementPlacement Reservation = Placement;
345 Reservation.Rows = RowCount;
346 Reservation.Cols = MaxSignatureCols;
347 return Reservation;
348}
349
350static bool reserveClipCullRows(MutableArrayRef<SignatureRow> Rows,
351 unsigned StartRow,
352 const ElementPlacement &Reservation) {
353 std::optional<uint8_t> ColumnMask = canPlaceAt(Rows, StartRow, Placement: Reservation);
354 if (!ColumnMask)
355 return false;
356 placeRowsAt(Rows, StartRow, Placement: Reservation, ColumnMask: *ColumnMask);
357 return true;
358}
359
360static std::optional<unsigned>
361reserveNextClipCullRows(MutableArrayRef<SignatureRow> Rows,
362 const ElementPlacement &Reservation) {
363 for (unsigned StartRow = 0; StartRow != Rows.size(); ++StartRow)
364 if (reserveClipCullRows(Rows, StartRow, Reservation))
365 return StartRow;
366 return std::nullopt;
367}
368
369// Reserves the whole signature rows that back the clip/cull grid rows that the
370// element is packed into. Existing rows cannot be moved without breaking
371// prefix stability, so an indexed element requires them to be adjacent.
372static std::optional<SignaturePackingError::ErrorKind>
373reserveClipCullSignatureRows(MutableArrayRef<SignatureRow> SignatureRows,
374 ClipCullState &State,
375 const ElementPlacement &Placement,
376 unsigned NewRowsUsed) {
377 if (Placement.Rows == 1) {
378 const ElementPlacement Reservation = getClipCullReservation(Placement, RowCount: 1);
379 for (unsigned Row = State.RowsUsed; Row < NewRowsUsed; ++Row) {
380 std::optional<unsigned> StartRow =
381 reserveNextClipCullRows(Rows: SignatureRows, Reservation);
382 if (!StartRow)
383 return SignaturePackingError::SignatureOverflow;
384 State.SignatureRows[Row] = *StartRow;
385 }
386 return std::nullopt;
387 }
388
389 if (State.RowsUsed == 0) {
390 std::optional<unsigned> StartRow = reserveNextClipCullRows(
391 Rows: SignatureRows, Reservation: getClipCullReservation(Placement, RowCount: MaxClipCullRows));
392 if (!StartRow)
393 return SignaturePackingError::SignatureOverflow;
394 State.SignatureRows[0] = *StartRow;
395 State.SignatureRows[1] = *StartRow + 1;
396 return std::nullopt;
397 }
398
399 if (State.RowsUsed == 1) {
400 const unsigned StartRow = State.SignatureRows[0] + 1;
401 if (StartRow >= SignatureRows.size())
402 return SignaturePackingError::SignatureOverflow;
403 if (!reserveClipCullRows(Rows: SignatureRows, StartRow,
404 Reservation: getClipCullReservation(Placement, RowCount: 1)))
405 return SignaturePackingError::ClipCullNotAdjacent;
406 State.SignatureRows[1] = StartRow;
407 return std::nullopt;
408 }
409
410 if (State.SignatureRows[0] + 1 != State.SignatureRows[1])
411 return SignaturePackingError::ClipCullNotAdjacent;
412 return std::nullopt;
413}
414
415static std::optional<SignaturePackingError::ErrorKind>
416packClipCullElement(ElementLocation &Location,
417 MutableArrayRef<SignatureRow> SignatureRows,
418 ClipCullState &State, const ElementPlacement &Placement) {
419 std::optional<uint8_t> ColumnMask;
420 unsigned ClipCullStartRow = 0;
421 while (ClipCullStartRow + Placement.Rows <= MaxClipCullRows) {
422 ColumnMask = canPlaceAt(Rows: State.Rows, StartRow: ClipCullStartRow, Placement);
423 if (ColumnMask)
424 break;
425 ++ClipCullStartRow;
426 }
427 if (!ColumnMask)
428 return SignaturePackingError::ClipCullOverflow;
429
430 const unsigned NewRowsUsed = ClipCullStartRow + Placement.Rows;
431 if (std::optional<SignaturePackingError::ErrorKind> Kind =
432 reserveClipCullSignatureRows(SignatureRows, State, Placement,
433 NewRowsUsed))
434 return Kind;
435
436 placeRowsAt(Rows: State.Rows, StartRow: ClipCullStartRow, Placement, ColumnMask: *ColumnMask);
437 State.RowsUsed = std::max(a: State.RowsUsed, b: NewRowsUsed);
438 Location.Row = State.SignatureRows[ClipCullStartRow];
439 Location.Col = getStartColumn(ColumnMask: *ColumnMask);
440 return std::nullopt;
441}
442
443// Work only on scratch rows and locations. The caller commits the entire
444// clip/cull phase after every stream succeeds.
445static Error
446packOptimizedClipCullStream(MutableArrayRef<OptimizedClipCullElement> Elements,
447 MutableArrayRef<SignatureRow> Rows) {
448 if (Elements.empty())
449 return Error::success();
450
451 std::array<SignatureRow, MaxClipCullRows> LocalRows;
452 bool HasIndexed = false;
453 for (auto &Element : Elements) {
454 if (!prefixPackElement(Location&: Element.Location, Rows: LocalRows, Placement: Element.Placement))
455 return make_error<SignaturePackingError>(
456 Args: SignaturePackingError::ClipCullOverflow, Args&: Element.Index);
457 HasIndexed |= Element.Placement.Rows > 1;
458 }
459
460 if (!HasIndexed) {
461 // Keep single-row elements grouped into at most two rows, rather than
462 // scattering individual distances across unrelated signature gaps.
463 std::array<ElementLocation, MaxClipCullRows> Destinations;
464 for (unsigned Row = 0; Row != MaxClipCullRows; ++Row) {
465 auto First = llvm::find_if(Range&: Elements, P: [Row](const auto &Element) {
466 return Element.Location.Row == Row;
467 });
468 if (First == Elements.end())
469 continue;
470 ElementPlacement Bundle = First->Placement;
471 Bundle.Cols = popcount(Value: LocalRows[Row].OccupiedColumns);
472 if (!prefixPackElement(Location&: Destinations[Row], Rows, Placement: Bundle))
473 return make_error<SignaturePackingError>(
474 Args: SignaturePackingError::SignatureOverflow, Args&: First->Index);
475 // Later elements can establish an initially undefined interpolation mode.
476 Rows[Destinations[Row].Row].InterpMode = LocalRows[Row].InterpMode;
477 }
478 for (auto &Element : Elements) {
479 ElementLocation Destination = Destinations[Element.Location.Row];
480 Element.Location = {
481 .Row: Destination.Row,
482 .Col: static_cast<uint8_t>(Destination.Col + Element.Location.Col)};
483 }
484 return Error::success();
485 }
486
487 // Try the whole indexed group in each adjacent pair. Preserve absolute row
488 // coordinates for existing indexed ranges, even those extending beyond it.
489 for (unsigned Row = 0; Row + MaxClipCullRows <= Rows.size(); ++Row) {
490 SmallVector<SignatureRow, MaxSignatureRows> CandidateRows(Rows.begin(),
491 Rows.end());
492 bool Fits = true;
493 for (auto &Element : Elements) {
494 Element.Location = {};
495 for (unsigned Start = Row;
496 Start + Element.Placement.Rows <= Row + MaxClipCullRows; ++Start) {
497 if (auto Mask = canPlaceAt(Rows: CandidateRows, StartRow: Start, Placement: Element.Placement)) {
498 placeAt(Rows: CandidateRows, StartRow: Start, Placement: Element.Placement, ColumnMask: *Mask,
499 Location&: Element.Location);
500 break;
501 }
502 }
503 if (Element.Location.Row == UnallocatedRow) {
504 Fits = false;
505 break;
506 }
507 }
508 if (Fits) {
509 llvm::copy(Range: ArrayRef(CandidateRows).slice(N: Row, M: MaxClipCullRows),
510 Out: Rows.begin() + Row);
511 return Error::success();
512 }
513 }
514 // No element alone necessarily caused this failure; identify the group.
515 return make_error<SignaturePackingError>(
516 Args: SignaturePackingError::SignatureOverflow, Args&: Elements.front().Index);
517}
518
519static Expected<unsigned>
520packOptimizedClipCull(MutableArrayRef<SemanticSignatureElement> Elements,
521 ArrayRef<unsigned> Order,
522 MutableArrayRef<SignatureRows> Rows,
523 bool UseNative16BitTypes) {
524 if (Order.empty())
525 return 0;
526 SmallVector<SmallVector<OptimizedClipCullElement>, MaxGeometryStreams>
527 Streams(Rows.size());
528 for (unsigned Index : Order) {
529 const auto &Element = Elements[Index];
530 assert(Element.StartRow == UnallocatedRow &&
531 Element.StartCol == UnallocatedCol && "already allocated?");
532 assert(Element.Rows > 0 && "signature element must have at least one row");
533 assert(Element.Cols > 0 && Element.Cols <= MaxSignatureCols &&
534 "signature element must have between 1 and 4 columns");
535 if (Element.GSStream >= Rows.size())
536 return make_error<SignaturePackingError>(
537 Args: SignaturePackingError::InvalidGeometryStream, Args&: Index);
538 Streams[Element.GSStream].push_back(
539 Elt: {.Index: Index,
540 .Placement: getElementPlacement(Element, Interpretation: SemanticInterpretation::ClipCull,
541 UseNative16BitTypes),
542 .Location: {}});
543 }
544
545 SmallVector<SignatureRows, MaxGeometryStreams> CandidateRows(Rows.begin(),
546 Rows.end());
547 for (unsigned Stream = 0; Stream != Streams.size(); ++Stream)
548 if (Error Err =
549 packOptimizedClipCullStream(Elements: Streams[Stream], Rows: CandidateRows[Stream]))
550 return std::move(Err);
551
552 // Publish only after every stream succeeds. No rollback or partial-prefix
553 // recovery is needed, and preceding non-clip/cull allocations remain intact.
554 llvm::copy(Range&: CandidateRows, Out: Rows.begin());
555 unsigned NumRows = 0;
556 for (const auto &Stream : Streams)
557 for (const auto &Element : Stream) {
558 Elements[Element.Index].StartRow = Element.Location.Row;
559 Elements[Element.Index].StartCol = Element.Location.Col;
560 NumRows =
561 std::max(a: NumRows, b: Element.Location.Row + Element.Placement.Rows);
562 }
563 return NumRows;
564}
565
566void SignaturePackingError::log(raw_ostream &OS) const {
567 switch (Kind) {
568 case SignatureOverflow:
569 OS << "signature elements do not fit in " << MaxSignatureRows << " rows";
570 break;
571 case SemanticIndexOutOfRange:
572 OS << "semantic index must be less than " << MaxSignatureRows;
573 break;
574 case ClipCullOverflow:
575 OS << "clip/cull elements do not fit in " << MaxClipCullRows << " rows";
576 break;
577 case ClipCullNotAdjacent:
578 OS << "indexed clip/cull elements require adjacent signature rows";
579 break;
580 case InvalidGeometryStream:
581 OS << "signature element has an invalid geometry stream: expected an index "
582 "less than "
583 << MaxGeometryStreams << " for geometry outputs, or zero otherwise";
584 break;
585 }
586 OS << " (element " << ElementIndex << ")";
587}
588
589Expected<unsigned> llvm::hlsl::packSignatureStacked(
590 MutableArrayRef<SemanticSignatureElement> Elements,
591 Triple::EnvironmentType ShaderStage, IOType IOTy) {
592 assert(ShaderStage == Triple::Vertex && IOTy == IOType::In &&
593 "stacked packing is only valid for a vertex shader input signature");
594
595 unsigned NextRow = 0;
596 for (auto &&[Index, Element] : enumerate(First&: Elements)) {
597 assert(Element.StartRow == UnallocatedRow &&
598 Element.StartCol == UnallocatedCol && "already allocated?");
599 assert(Element.Rows > 0 && "signature element must have at least one row");
600 assert(Element.Cols > 0 && Element.Cols <= MaxSignatureCols &&
601 "signature element must have between 1 and 4 columns");
602
603 SemanticInterpretation Interpretation =
604 getInterpretationKind(SemanticKind: Element.SemanticKind, ShaderStage, IOTy);
605 if (Interpretation == SemanticInterpretation::NotAllocated)
606 continue;
607
608 assert((Interpretation == SemanticInterpretation::Arbitrary ||
609 Interpretation == SemanticInterpretation::SV ||
610 Interpretation == SemanticInterpretation::SGV) &&
611 "unexpected semantic interpretation for stacked packing, should "
612 "have been diagnosed by Sema");
613
614 if (Element.Rows > MaxSignatureRows - NextRow)
615 return make_error<SignaturePackingError>(
616 Args: SignaturePackingError::SignatureOverflow,
617 Args: static_cast<unsigned>(Index));
618
619 Element.StartRow = NextRow;
620 Element.StartCol = 0;
621 NextRow += Element.Rows;
622 }
623
624 return NextRow;
625}
626
627template <typename IndexRange>
628static Expected<unsigned> packSignatureInOrder(
629 MutableArrayRef<SemanticSignatureElement> Elements, const IndexRange &Order,
630 Triple::EnvironmentType ShaderStage, IOType IOTy, bool UseNative16BitTypes,
631 MutableArrayRef<SignatureRows> Rows) {
632 SmallVector<ClipCullState, 1> ClipCullStates(Rows.size());
633 unsigned NumRows = 0;
634 for (unsigned Index : Order) {
635 const SemanticSignatureElement &Element = Elements[Index];
636 assert(Element.StartRow == UnallocatedRow &&
637 Element.StartCol == UnallocatedCol && "already allocated?");
638 assert(Element.Rows > 0 && "signature element must have at least one row");
639 assert(Element.Cols > 0 && Element.Cols <= MaxSignatureCols &&
640 "signature element must have between 1 and 4 columns");
641 if (Element.GSStream >= Rows.size())
642 return make_error<SignaturePackingError>(
643 Args: SignaturePackingError::InvalidGeometryStream,
644 Args: static_cast<unsigned>(Index));
645
646 SemanticInterpretation Interpretation =
647 getInterpretationKind(SemanticKind: Element.SemanticKind, ShaderStage, IOTy);
648 if (Interpretation == SemanticInterpretation::NotAllocated)
649 continue;
650
651 assert((Interpretation == SemanticInterpretation::Arbitrary ||
652 Interpretation == SemanticInterpretation::SV ||
653 Interpretation == SemanticInterpretation::SGV ||
654 Interpretation == SemanticInterpretation::ClipCull ||
655 Interpretation == SemanticInterpretation::TessFactor) &&
656 "unexpected semantic interpretation for prefix-stable packing, "
657 "should have been diagnosed by Sema");
658
659 const ElementPlacement Placement =
660 getElementPlacement(Element, Interpretation, UseNative16BitTypes);
661
662 const unsigned StreamIndex = Element.GSStream;
663 MutableArrayRef<SignatureRow> StreamRows = Rows[StreamIndex];
664 ElementLocation Location;
665
666 if (Interpretation == SemanticInterpretation::ClipCull) {
667 if (std::optional<SignaturePackingError::ErrorKind> Kind =
668 packClipCullElement(Location, SignatureRows: StreamRows,
669 State&: ClipCullStates[StreamIndex], Placement))
670 return make_error<SignaturePackingError>(Args&: *Kind, Args&: Index);
671 } else if (!prefixPackElement(Location, Rows: StreamRows, Placement)) {
672 return make_error<SignaturePackingError>(
673 Args: SignaturePackingError::SignatureOverflow,
674 Args: static_cast<unsigned>(Index));
675 }
676
677 Elements[Index].StartRow = Location.Row;
678 Elements[Index].StartCol = Location.Col;
679 NumRows = std::max(a: NumRows, b: Location.Row + Element.Rows);
680 }
681
682 return NumRows;
683}
684
685Expected<unsigned> llvm::hlsl::packSignaturePrefixStable(
686 MutableArrayRef<SemanticSignatureElement> Elements,
687 Triple::EnvironmentType ShaderStage, IOType IOTy,
688 bool UseNative16BitTypes) {
689 assert(!(ShaderStage == Triple::Vertex && IOTy == IOType::In) &&
690 !(ShaderStage == Triple::Pixel && IOTy == IOType::Out) &&
691 "prefix-stable packing is not valid for vertex inputs or pixel "
692 "outputs");
693 const unsigned StreamCount =
694 ShaderStage == Triple::Geometry && IOTy == IOType::Out
695 ? MaxGeometryStreams
696 : 1;
697 SmallVector<SignatureRows, 1> Rows(StreamCount);
698 return packSignatureInOrder(Elements, Order: llvm::seq<unsigned>(Begin: 0, End: Elements.size()),
699 ShaderStage, IOTy, UseNative16BitTypes, Rows);
700}
701
702Expected<unsigned> llvm::hlsl::packSignatureIndexed(
703 MutableArrayRef<SemanticSignatureElement> Elements,
704 Triple::EnvironmentType ShaderStage, IOType IOTy) {
705 assert(ShaderStage == Triple::Pixel && IOTy == IOType::Out &&
706 "indexed packing is only valid for a pixel shader output signature");
707
708 static_assert(MaxSignatureRows <= std::numeric_limits<uint32_t>::digits,
709 "row allocation mask is too small");
710 [[maybe_unused]] uint32_t AllocatedRows = 0;
711 unsigned NumRows = 0;
712 for (auto &&[Index, Element] : enumerate(First&: Elements)) {
713 assert(Element.StartRow == UnallocatedRow &&
714 Element.StartCol == UnallocatedCol && "already allocated?");
715 assert(Element.Rows > 0 && "signature element must have at least one row");
716 assert(Element.Cols > 0 && Element.Cols <= MaxSignatureCols &&
717 "signature element must have between 1 and 4 columns");
718
719 SemanticInterpretation Interpretation =
720 getInterpretationKind(SemanticKind: Element.SemanticKind, ShaderStage, IOTy);
721 if (Interpretation == SemanticInterpretation::NotAllocated)
722 continue;
723
724 assert(Interpretation == SemanticInterpretation::Target &&
725 "unexpected semantic interpretation for indexed packing, should "
726 "have been diagnosed by Sema");
727 assert(Element.Rows == 1 && Element.SemanticIndices.size() == 1 &&
728 "target elements must occupy one semantic row");
729
730 const uint32_t Row = Element.SemanticIndices.front();
731 if (Row >= MaxSignatureRows)
732 return make_error<SignaturePackingError>(
733 Args: SignaturePackingError::SemanticIndexOutOfRange,
734 Args: static_cast<unsigned>(Index));
735
736 const uint32_t RowMask = uint32_t{1} << Row;
737 assert(!(AllocatedRows & RowMask) &&
738 "target semantic indices must be unique, verified in SemaHLSL");
739 AllocatedRows |= RowMask;
740
741 Element.StartRow = Row;
742 Element.StartCol = 0;
743 NumRows = std::max(a: NumRows, b: Row + 1);
744 }
745
746 return NumRows;
747}
748
749Expected<unsigned> llvm::hlsl::packSignatureOptimized(
750 MutableArrayRef<SemanticSignatureElement> Elements,
751 Triple::EnvironmentType ShaderStage, IOType IOTy,
752 bool UseNative16BitTypes) {
753 assert(!(ShaderStage == Triple::Vertex && IOTy == IOType::In) &&
754 !(ShaderStage == Triple::Pixel && IOTy == IOType::Out) &&
755 "optimized packing is not valid for vertex inputs or pixel outputs");
756
757 struct SortKey {
758 PackingGroup Group;
759 dxbc::PSV::InterpolationMode InterpMode;
760 uint32_t Rows;
761 uint8_t Cols;
762 uint32_t SigId;
763 unsigned OriginalIndex;
764 };
765 SmallVector<SortKey> SortedKeys;
766 SortedKeys.reserve(N: Elements.size());
767 for (auto [Index, Element] : enumerate(First&: Elements))
768 SortedKeys.push_back(Elt: {.Group: getOptimizedPackingGroup(Element, ShaderStage, IOTy),
769 .InterpMode: Element.InterpMode, .Rows: Element.Rows, .Cols: Element.Cols,
770 .SigId: Element.SigId, .OriginalIndex: static_cast<unsigned>(Index)});
771
772 llvm::sort(C&: SortedKeys, Comp: [](const SortKey &Left, const SortKey &Right) {
773 if (Left.Group != Right.Group)
774 return Left.Group < Right.Group;
775 if (Left.InterpMode != Right.InterpMode)
776 return Left.InterpMode < Right.InterpMode;
777 if (Left.Rows != Right.Rows)
778 return Left.Rows > Right.Rows;
779 if (Left.Cols != Right.Cols)
780 return Left.Cols > Right.Cols;
781 return Left.SigId < Right.SigId;
782 });
783
784 const unsigned StreamCount =
785 ShaderStage == Triple::Geometry && IOTy == IOType::Out
786 ? MaxGeometryStreams
787 : 1;
788 SmallVector<SignatureRows, 1> Rows(StreamCount);
789 auto ClipBegin = llvm::partition_point(Range&: SortedKeys, P: [](const SortKey &Key) {
790 return Key.Group < PackingGroup::ClipCull;
791 });
792 auto ClipEnd = llvm::partition_point(
793 Range: make_range(x: ClipBegin, y: SortedKeys.end()),
794 P: [](const SortKey &Key) { return Key.Group == PackingGroup::ClipCull; });
795
796 auto Pack = [&](auto Begin, auto End) {
797 auto Order = map_range(make_range(Begin, End), [](const SortKey &Key) {
798 return Key.OriginalIndex;
799 });
800 return packSignatureInOrder(Elements, Order, ShaderStage, IOTy,
801 UseNative16BitTypes, Rows);
802 };
803
804 Expected<unsigned> Before = Pack(SortedKeys.begin(), ClipBegin);
805 if (!Before)
806 return Before.takeError();
807 SmallVector<unsigned> ClipCullOrder;
808 for (const SortKey &Key : make_range(x: ClipBegin, y: ClipEnd))
809 ClipCullOrder.push_back(Elt: Key.OriginalIndex);
810 Expected<unsigned> ClipCull =
811 packOptimizedClipCull(Elements, Order: ClipCullOrder, Rows, UseNative16BitTypes);
812 if (!ClipCull)
813 return ClipCull.takeError();
814 Expected<unsigned> After = Pack(ClipEnd, SortedKeys.end());
815 if (!After)
816 return After.takeError();
817 return std::max(l: {*Before, *ClipCull, *After});
818}
819