drm/amdkfd: Move gfx12 trap handler to separate file

 *   cpp -DASIC_FAMILY=CHIP_PLUM_BONITO cwsr_trap_handler_gfx10.asm -P -o gfx11.sp3

 *   sp3 gfx11.sp3 -hex gfx11.hex

 *

 * gfx12:

 *   cpp -DASIC_FAMILY=CHIP_GFX12 cwsr_trap_handler_gfx10.asm -P -o gfx12.sp3

 *   sp3 gfx12.sp3 -hex gfx12.hex

 */

#define CHIP_NAVI10 26

#define CHIP_SIENNA_CICHLID 30

#define CHIP_PLUM_BONITO 36

#define CHIP_GFX12 37

#define NO_SQC_STORE (ASIC_FAMILY >= CHIP_SIENNA_CICHLID)

#define HAVE_XNACK (ASIC_FAMILY < CHIP_SIENNA_CICHLID)

#define HAVE_SENDMSG_RTN (ASIC_FAMILY >= CHIP_PLUM_BONITO)

#define HAVE_BUFFER_LDS_LOAD (ASIC_FAMILY < CHIP_PLUM_BONITO)

#define SW_SA_TRAP (ASIC_FAMILY >= CHIP_PLUM_BONITO && ASIC_FAMILY < CHIP_GFX12)

#define SW_SA_TRAP (ASIC_FAMILY == CHIP_PLUM_BONITO)

#define SAVE_AFTER_XNACK_ERROR (HAVE_XNACK && !NO_SQC_STORE) // workaround for TCP store failure after XNACK error when ALLOW_REPLAY=0, for debugger

#define SINGLE_STEP_MISSED_WORKAROUND 1	//workaround for lost MODE.DEBUG_EN exception when SAVECTX raised

#if ASIC_FAMILY < CHIP_GFX12

#define S_COHERENCE glc:1

#define V_COHERENCE slc:1 glc:1

#define S_WAITCNT_0 s_waitcnt 0

#else

#define S_COHERENCE scope:SCOPE_SYS

#define V_COHERENCE scope:SCOPE_SYS

#define S_WAITCNT_0 s_wait_idle

#define HW_REG_SHADER_FLAT_SCRATCH_LO HW_REG_WAVE_SCRATCH_BASE_LO

#define HW_REG_SHADER_FLAT_SCRATCH_HI HW_REG_WAVE_SCRATCH_BASE_HI

#define HW_REG_GPR_ALLOC HW_REG_WAVE_GPR_ALLOC

#define HW_REG_LDS_ALLOC HW_REG_WAVE_LDS_ALLOC

#define HW_REG_MODE HW_REG_WAVE_MODE

#endif

#if ASIC_FAMILY < CHIP_GFX12

var SQ_WAVE_STATUS_SPI_PRIO_MASK		= 0x00000006

var SQ_WAVE_STATUS_HALT_MASK			= 0x2000

var SQ_WAVE_STATUS_ECC_ERR_MASK			= 0x20000

var S_STATUS_HALT_MASK				= SQ_WAVE_STATUS_HALT_MASK

var S_SAVE_PC_HI_TRAP_ID_MASK			= 0x00FF0000

var S_SAVE_PC_HI_HT_MASK			= 0x01000000

#else

var SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_MASK	= 0x4

var SQ_WAVE_STATE_PRIV_SCC_SHIFT		= 9

var SQ_WAVE_STATE_PRIV_SYS_PRIO_MASK		= 0xC00

var SQ_WAVE_STATE_PRIV_HALT_MASK		= 0x4000

var SQ_WAVE_STATE_PRIV_POISON_ERR_MASK		= 0x8000

var SQ_WAVE_STATE_PRIV_POISON_ERR_SHIFT		= 15

var SQ_WAVE_STATUS_WAVE64_SHIFT			= 29

var SQ_WAVE_STATUS_WAVE64_SIZE			= 1

var SQ_WAVE_LDS_ALLOC_GRANULARITY		= 9

var S_STATUS_HWREG				= HW_REG_WAVE_STATE_PRIV

var S_STATUS_ALWAYS_CLEAR_MASK			= SQ_WAVE_STATE_PRIV_SYS_PRIO_MASK|SQ_WAVE_STATE_PRIV_POISON_ERR_MASK

var S_STATUS_HALT_MASK				= SQ_WAVE_STATE_PRIV_HALT_MASK

var S_SAVE_PC_HI_TRAP_ID_MASK			= 0xF0000000

#endif

var SQ_WAVE_STATUS_NO_VGPRS_SHIFT		= 24

var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT		= 12

var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT		= 12

#endif

#if ASIC_FAMILY < CHIP_GFX12

var SQ_WAVE_TRAPSTS_SAVECTX_MASK		= 0x400

var SQ_WAVE_TRAPSTS_EXCP_MASK			= 0x1FF

var SQ_WAVE_TRAPSTS_SAVECTX_SHIFT		= 10

var S_TRAPSTS_HWREG				= HW_REG_TRAPSTS

var S_TRAPSTS_SAVE_CONTEXT_MASK			= SQ_WAVE_TRAPSTS_SAVECTX_MASK

var S_TRAPSTS_SAVE_CONTEXT_SHIFT		= SQ_WAVE_TRAPSTS_SAVECTX_SHIFT

#else

var SQ_WAVE_EXCP_FLAG_PRIV_ADDR_WATCH_MASK	= 0xF

var SQ_WAVE_EXCP_FLAG_PRIV_MEM_VIOL_MASK	= 0x10

var SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_SHIFT	= 5

var SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_MASK	= 0x20

var SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_MASK	= 0x40

var SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_SHIFT	= 6

var SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK	= 0x80

var SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_SHIFT	= 7

var SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_MASK	= 0x100

var SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_SHIFT	= 8

var SQ_WAVE_EXCP_FLAG_PRIV_WAVE_END_MASK	= 0x200

var SQ_WAVE_EXCP_FLAG_PRIV_TRAP_AFTER_INST_MASK	= 0x800

var SQ_WAVE_TRAP_CTRL_ADDR_WATCH_MASK		= 0x80

var SQ_WAVE_TRAP_CTRL_TRAP_AFTER_INST_MASK	= 0x200

var S_TRAPSTS_HWREG				= HW_REG_WAVE_EXCP_FLAG_PRIV

var S_TRAPSTS_SAVE_CONTEXT_MASK			= SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_MASK

var S_TRAPSTS_SAVE_CONTEXT_SHIFT		= SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_SHIFT

var S_TRAPSTS_NON_MASKABLE_EXCP_MASK		= SQ_WAVE_EXCP_FLAG_PRIV_MEM_VIOL_MASK		|\

						  SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_MASK	|\

						  SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK		|\

						  SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_MASK	|\

						  SQ_WAVE_EXCP_FLAG_PRIV_WAVE_END_MASK		|\

						  SQ_WAVE_EXCP_FLAG_PRIV_TRAP_AFTER_INST_MASK

var S_TRAPSTS_RESTORE_PART_1_SIZE		= SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_SHIFT

var S_TRAPSTS_RESTORE_PART_2_SHIFT		= SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_SHIFT

var S_TRAPSTS_RESTORE_PART_2_SIZE		= SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_SHIFT - SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_SHIFT

var S_TRAPSTS_RESTORE_PART_3_SHIFT		= SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_SHIFT

var S_TRAPSTS_RESTORE_PART_3_SIZE		= 32 - S_TRAPSTS_RESTORE_PART_3_SHIFT

var BARRIER_STATE_SIGNAL_OFFSET			= 16

var BARRIER_STATE_VALID_OFFSET			= 0

#endif

// bits [31:24] unused by SPI debug data

var TTMP11_SAVE_REPLAY_W64H_SHIFT		= 31

L_HALTED:

	// Host trap may occur while wave is halted.

#if ASIC_FAMILY < CHIP_GFX12

	s_and_b32	ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK

#else

	s_and_b32	ttmp2, s_save_trapsts, SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK

#endif

	s_cbranch_scc1	L_FETCH_2ND_TRAP

L_CHECK_SAVE:

	// Check for maskable exceptions in trapsts.excp and trapsts.excp_hi.

	// Maskable exceptions only cause the wave to enter the trap handler if

	// their respective bit in mode.excp_en is set.

#if ASIC_FAMILY < CHIP_GFX12

	s_and_b32	ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_EXCP_MASK|SQ_WAVE_TRAPSTS_EXCP_HI_MASK

	s_cbranch_scc0	L_CHECK_TRAP_ID

	s_lshl_b32	ttmp2, ttmp2, SQ_WAVE_MODE_EXCP_EN_SHIFT

	s_and_b32	ttmp2, ttmp2, ttmp3

	s_cbranch_scc1	L_FETCH_2ND_TRAP

#else

	s_getreg_b32	ttmp2, hwreg(HW_REG_WAVE_EXCP_FLAG_USER)

	s_and_b32	ttmp3, s_save_trapsts, SQ_WAVE_EXCP_FLAG_PRIV_ADDR_WATCH_MASK

	s_cbranch_scc0	L_NOT_ADDR_WATCH

	s_or_b32	ttmp2, ttmp2, SQ_WAVE_TRAP_CTRL_ADDR_WATCH_MASK

L_NOT_ADDR_WATCH:

	s_getreg_b32	ttmp3, hwreg(HW_REG_WAVE_TRAP_CTRL)

	s_and_b32	ttmp2, ttmp3, ttmp2

	s_cbranch_scc1	L_FETCH_2ND_TRAP

#endif

L_CHECK_TRAP_ID:

	// Check trap_id != 0

#if SINGLE_STEP_MISSED_WORKAROUND

	// Prioritize single step exception over context save.

	// Second-level trap will halt wave and RFE, re-entering for SAVECTX.

#if ASIC_FAMILY < CHIP_GFX12

	s_getreg_b32	ttmp2, hwreg(HW_REG_MODE)

	s_and_b32	ttmp2, ttmp2, SQ_WAVE_MODE_DEBUG_EN_MASK

#else

	// WAVE_TRAP_CTRL is already in ttmp3.

	s_and_b32	ttmp3, ttmp3, SQ_WAVE_TRAP_CTRL_TRAP_AFTER_INST_MASK

#endif

	s_cbranch_scc1	L_FETCH_2ND_TRAP

#endif

	s_cbranch_scc1	L_TRAP_CASE

	// Host trap will not cause trap re-entry.

#if ASIC_FAMILY < CHIP_GFX12

	s_and_b32	ttmp2, s_save_pc_hi, S_SAVE_PC_HI_HT_MASK

#else

	s_getreg_b32	ttmp2, hwreg(HW_REG_WAVE_EXCP_FLAG_PRIV)

	s_and_b32	ttmp2, ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK

#endif

	s_cbranch_scc1	L_EXIT_TRAP

	s_or_b32	s_save_status, s_save_status, S_STATUS_HALT_MASK

	s_and_b64	exec, exec, exec					// Restore STATUS.EXECZ, not writable by s_setreg_b32

	s_and_b64	vcc, vcc, vcc						// Restore STATUS.VCCZ, not writable by s_setreg_b32

#if ASIC_FAMILY < CHIP_GFX12

	s_setreg_b32	hwreg(S_STATUS_HWREG), s_save_status

#else

	// STATE_PRIV.BARRIER_COMPLETE may have changed since we read it.

	// Only restore fields which the trap handler changes.

	s_lshr_b32	s_save_status, s_save_status, SQ_WAVE_STATE_PRIV_SCC_SHIFT

	s_setreg_b32	hwreg(S_STATUS_HWREG, SQ_WAVE_STATE_PRIV_SCC_SHIFT, \

		SQ_WAVE_STATE_PRIV_POISON_ERR_SHIFT - SQ_WAVE_STATE_PRIV_SCC_SHIFT + 1), s_save_status

#endif

	s_rfe_b64	[ttmp0, ttmp1]

L_SAVE:

	s_endpgm

L_HAVE_VGPRS:

#endif

#if ASIC_FAMILY >= CHIP_GFX12

	s_getreg_b32	s_save_tmp, hwreg(HW_REG_WAVE_STATUS)

	s_bitcmp1_b32	s_save_tmp, SQ_WAVE_STATUS_NO_VGPRS_SHIFT

	s_cbranch_scc0	L_HAVE_VGPRS

	s_endpgm

L_HAVE_VGPRS:

#endif

	s_and_b32	s_save_pc_hi, s_save_pc_hi, 0x0000ffff			//pc[47:32]

	s_mov_b32	s_save_tmp, 0

	s_setreg_b32	hwreg(S_TRAPSTS_HWREG, S_TRAPSTS_SAVE_CONTEXT_SHIFT, 1), s_save_tmp	//clear saveCtx bit

	s_mov_b32	m0, 0x0							//Next lane of v2 to write to

#endif

#if ASIC_FAMILY >= CHIP_GFX12

	// Ensure no further changes to barrier or LDS state.

	// STATE_PRIV.BARRIER_COMPLETE may change up to this point.

	s_barrier_signal	-2

	s_barrier_wait	-2

	// Re-read final state of BARRIER_COMPLETE field for save.

	s_getreg_b32	s_save_tmp, hwreg(S_STATUS_HWREG)

	s_and_b32	s_save_tmp, s_save_tmp, SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_MASK

	s_andn2_b32	s_save_status, s_save_status, SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_MASK

	s_or_b32	s_save_status, s_save_status, s_save_tmp

#endif

	write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)

	write_hwreg_to_mem(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset)

	s_andn2_b32	s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK

	s_getreg_b32	s_save_m0, hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI)

	write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)

#if ASIC_FAMILY >= CHIP_GFX12

	s_getreg_b32	s_save_m0, hwreg(HW_REG_WAVE_EXCP_FLAG_USER)

	write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)

	s_getreg_b32	s_save_m0, hwreg(HW_REG_WAVE_TRAP_CTRL)

	write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)

	s_getreg_b32	s_save_tmp, hwreg(HW_REG_WAVE_STATUS)

	write_hwreg_to_mem(s_save_tmp, s_save_buf_rsrc0, s_save_mem_offset)

	s_get_barrier_state s_save_tmp, -1

	s_wait_kmcnt (0)

	write_hwreg_to_mem(s_save_tmp, s_save_buf_rsrc0, s_save_mem_offset)

#endif

#if NO_SQC_STORE

	// Write HWREGs with 16 VGPR lanes. TTMPs occupy space after this.

	s_mov_b32       exec_lo, 0xFFFF

	s_and_b32	s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF	//lds_size is zero?

	s_cbranch_scc0	L_SAVE_LDS_DONE						//no lds used? jump to L_SAVE_DONE

#if ASIC_FAMILY < CHIP_GFX12

	s_barrier								//LDS is used? wait for other waves in the same TG

#endif

	s_and_b32	s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK

	s_cbranch_scc0	L_SAVE_LDS_DONE

	s_mov_b32	s_restore_buf_rsrc2, 0					//NUM_RECORDS initial value = 0 (in bytes)

	s_mov_b32	s_restore_buf_rsrc3, S_RESTORE_BUF_RSRC_WORD3_MISC

#if ASIC_FAMILY >= CHIP_GFX12

	// Save s_restore_spi_init_hi for later use.

	s_mov_b32 s_restore_spi_init_hi_save, s_restore_spi_init_hi

#endif

	//determine it is wave32 or wave64

	get_wave_size2(s_restore_size)

	// s_barrier with MODE.DEBUG_EN=1, STATUS.PRIV=1 incorrectly asserts debug exception.

	// Clear DEBUG_EN before and restore MODE after the barrier.

	s_setreg_imm32_b32	hwreg(HW_REG_MODE), 0

#if ASIC_FAMILY < CHIP_GFX12

	s_barrier								//barrier to ensure the readiness of LDS before access attemps from any other wave in the same TG

#endif

	/* restore HW registers */

L_RESTORE_HWREG:

	s_mov_b32	s_restore_buf_rsrc2, 0x1000000				//NUM_RECORDS in bytes

#if ASIC_FAMILY >= CHIP_GFX12

	// Restore s_restore_spi_init_hi before the saved value gets clobbered.

	s_mov_b32 s_restore_spi_init_hi, s_restore_spi_init_hi_save

#endif

	read_hwreg_from_mem(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset)

	read_hwreg_from_mem(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset)

	read_hwreg_from_mem(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset)

	s_setreg_b32	hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI), s_restore_flat_scratch

#if ASIC_FAMILY >= CHIP_GFX12

	read_hwreg_from_mem(s_restore_tmp, s_restore_buf_rsrc0, s_restore_mem_offset)

	S_WAITCNT_0

	s_setreg_b32	hwreg(HW_REG_WAVE_EXCP_FLAG_USER), s_restore_tmp

	read_hwreg_from_mem(s_restore_tmp, s_restore_buf_rsrc0, s_restore_mem_offset)

	S_WAITCNT_0

	s_setreg_b32	hwreg(HW_REG_WAVE_TRAP_CTRL), s_restore_tmp

	// Only the first wave needs to restore the workgroup barrier.

	s_and_b32	s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK

	s_cbranch_scc0	L_SKIP_BARRIER_RESTORE

	// Skip over WAVE_STATUS, since there is no state to restore from it

	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, 4

	read_hwreg_from_mem(s_restore_tmp, s_restore_buf_rsrc0, s_restore_mem_offset)

	S_WAITCNT_0

	s_bitcmp1_b32	s_restore_tmp, BARRIER_STATE_VALID_OFFSET

	s_cbranch_scc0	L_SKIP_BARRIER_RESTORE

	// extract the saved signal count from s_restore_tmp

	s_lshr_b32	s_restore_tmp, s_restore_tmp, BARRIER_STATE_SIGNAL_OFFSET

	// We need to call s_barrier_signal repeatedly to restore the signal

	// count of the work group barrier.  The member count is already

	// initialized with the number of waves in the work group.

L_BARRIER_RESTORE_LOOP:

	s_and_b32	s_restore_tmp, s_restore_tmp, s_restore_tmp

	s_cbranch_scc0	L_SKIP_BARRIER_RESTORE

	s_barrier_signal	-1

	s_add_i32	s_restore_tmp, s_restore_tmp, -1

	s_branch	L_BARRIER_RESTORE_LOOP

L_SKIP_BARRIER_RESTORE:

#endif

	s_mov_b32	m0, s_restore_m0

	s_mov_b32	exec_lo, s_restore_exec_lo

	s_mov_b32	exec_hi, s_restore_exec_hi

	s_setreg_b32	hwreg(S_STATUS_HWREG), s_restore_status			// SCC is included, which is changed by previous salu

#if ASIC_FAMILY >= CHIP_GFX12

	// Make barrier and LDS state visible to all waves in the group.

	// STATE_PRIV.BARRIER_COMPLETE may change after this point.

	s_barrier_signal	-2

	s_barrier_wait	-2

#endif

	s_rfe_b64	s_restore_pc_lo						//Return to the main shader program and resume execution

L_END_PGM:

end

function get_wave_size2(s_reg)

#if ASIC_FAMILY < CHIP_GFX12

	s_getreg_b32	s_reg, hwreg(HW_REG_IB_STS2,SQ_WAVE_IB_STS2_WAVE64_SHIFT,SQ_WAVE_IB_STS2_WAVE64_SIZE)

#else

	s_getreg_b32	s_reg, hwreg(HW_REG_WAVE_STATUS,SQ_WAVE_STATUS_WAVE64_SHIFT,SQ_WAVE_STATUS_WAVE64_SIZE)

#endif

	s_lshl_b32	s_reg, s_reg, S_WAVE_SIZE

end