diff -uNr a/m/MANIFEST.TXT b/m/MANIFEST.TXT --- a/m/MANIFEST.TXT 0f03021ae4e766b63be5efe5cf5e672fc90202022ca05772621f5299c9b808d2c8c2439b7fa555716cb33a6adb8351e582275ffbafdf6cc633e528522343720e +++ b/m/MANIFEST.TXT b93a4289f9decd7fc48508715afe4f4237a1edd72646fef53ebab319d50f16adf546473b023c657c5c6d16a47cf3dfa1ecfdee1f00e5796126b8a6c727292904 @@ -1,2 +1,3 @@ 586606 m_genesis "Genesis." 586747 errata_slaveirq "Fix of slave IRQ clearing." +586983 tlb_and_exc_speedup "Exc. handler fastpaths and TLB caching." diff -uNr a/m/cpustate.asm b/m/cpustate.asm --- a/m/cpustate.asm 6ceecb9422f3d01b04ed2ecb183fb96665ff969087e2967eccef2703337401afb4f0ec91e4b7efda625ae4534ba1c3daba539a1fc794b2e594d7404ddf72399c +++ b/m/cpustate.asm 1427e0c04c15c733aed037b1a56252c2f7f8ee7960171ab548b6ce1244320ea1e9b4aa00045d94a4478aad0e605e9206aa9e8b92581be252b3e08c893ed372fe @@ -76,6 +76,19 @@ ;----------------------------------------------------------------------------- ;----------------------------------------------------------------------------- +; XMM Regs used for TLB Caching: +;----------------------------------------------------------------------------- +%define Rd_E_Last_Tag xmm5 ; Last good Tag on reading Even vAddr +%define Rd_E_Last_PFN xmm6 ; Last good PFN on reading Even vAddr +%define Rd_O_Last_Tag xmm7 ; Last good Tag on reading Odd vAddr +%define Rd_O_Last_PFN xmm8 ; Last good PFN on reading Odd vAddr +%define Wr_E_Last_Tag xmm9 ; Last good Tag on writing Even vAddr +%define Wr_E_Last_PFN xmm10 ; Last good PFN on writing Even vAddr +%define Wr_O_Last_Tag xmm11 ; Last good Tag on writing Odd vAddr +%define Wr_O_Last_PFN xmm12 ; Last good PFN on writing Odd vAddr +;----------------------------------------------------------------------------- + +;----------------------------------------------------------------------------- ; Access to MIPS Registers that live in MCPU (Emulator State) : ;----------------------------------------------------------------------------- ; Refer to given MIPS special Reg: diff -uNr a/m/flags.asm b/m/flags.asm --- a/m/flags.asm a7d2be62abfd1319dbafaa5f8eedd8718c3364a59d89e89c611d0a95d04ba6d81d089e93a65853f85ff3fdd45b413c2c582cbe9e7b082e40f18f206d2cafdbb1 +++ b/m/flags.asm 7dedb8135f032539dd5f6c0133070aff4078cadb322fcb7e665a56bdfe7940e13d463b2803ea9c9726e7606579218ba9684d21ae106890c6ef7a285119887364 @@ -23,8 +23,8 @@ %define RunningDelaySlot 1 %define Waiting 2 %define LL_Bit 3 -%define IsWriting 4 -%define ExcWasTLBNoMatch 5 +%define TLB_Rd_Cache_Valid 4 +%define TLB_Wr_Cache_Valid 5 %define Shutdown 6 ;----------------------------------------------------------------------------- ; Set a given Flag: diff -uNr a/m/i_decode.asm b/m/i_decode.asm --- a/m/i_decode.asm 46635728670946901edca4062d650365295341cf87401ac4fbb7f78423faf76b5c246a07ea8e5fc76fcd4b49ff669102f556f47f7c5a247670fd00ffee665b80 +++ b/m/i_decode.asm ad646cee418f8c037cda4cb8db4838733687e9d522537fcf080e044199b5b26daefa7ecdac572e0e2a606f46492130cde66a58ad33f12967809fafd949a8d524 @@ -261,16 +261,35 @@ ;----------------------------------------------------------------------------- ;----------------------------------------------------------------------------- -; Denote privileged (permitted in Kernel-Mode strictly) instructions. +; Jump to given target if interrupts are DISABLED: ;----------------------------------------------------------------------------- -%macro PRIVILEGED 0 +%macro JMP_IF_IRQS_OFF 1 + mov eax, CP0_Status ; eax := CP0_Status + and eax, 3 ; select only IE | EXL | ERL bits + sub eax, 1 ; if eax == 1, then interrupts enabled + jnz %1 ; so if interrupts not enabled, jump. +%endmacro +;----------------------------------------------------------------------------- + +;----------------------------------------------------------------------------- +; Jump to given target if CPU is currently in Kernel Mode: +;----------------------------------------------------------------------------- +%macro JMP_IF_KERNELMODE 1 bt CP0_Status, CP0St_UM ; CF := CP0St_UM Flag - jnc %%proceed ; If UM = 0: Kernel Mode, proceed. + jnc %1 ; If UM = 0: Kernel Mode, proceed. test CP0_Status, (1 << CP0St_EXL) | (1 << CP0St_ERL) ; EXL or ERL - jnz %%proceed ; If EXL && ERL: Kernel Mode, proceed. + jnz %1 ; If EXL or ERL: Kernel Mode, proceed. +%endmacro +;----------------------------------------------------------------------------- + +;----------------------------------------------------------------------------- +; Denote privileged (permitted in Kernel-Mode strictly) instructions. +;----------------------------------------------------------------------------- +%macro PRIVILEGED 0 + JMP_IF_KERNELMODE %%proceed ; If in Kernel Mode : proceed; ;; We are NOT in kernel mode, but trying to execute a privileged inst: SetEXC EXC_RI ; Set the 'Reserved Instr.' Exception. - jmp _Handle_Exception ; Go straight to exception handler. + jmp _Handle_Exception_Other ; Go straight to exception handler. %%proceed ; We're in Kernel Mode, so proceed with the privileged instruction. %endmacro ;----------------------------------------------------------------------------- diff -uNr a/m/knobs.asm b/m/knobs.asm --- a/m/knobs.asm e75680eee6b4d6dab5e13fd02db2a86702136633846d4e9d9ca17ffaae25ce6c1d0d138db69081802520d9b418b7027a8150271e15e954971ba44d2506f70ad1 +++ b/m/knobs.asm 3b8e7b9cf4b6b37a941b53f534fa000b523941e5c52747f0ccf92397c5e64fdcf74bbdd241e70e51bef8893954c0cf5f4db5a89066b68349a3de4f24f737bdbc @@ -23,6 +23,21 @@ ;----------------------------------------------------------------------------- ;----------------------------------------------------------------------------- +; If TLBWR_CHEAT is enabled, the TLBWR ('Write Random TLB Entry') instruction +; will slide all unwired entries down by one slot and write into the freed +; slot near the top permitted by CP0_Wired, instead of the traditional +; behaviour (where entry indexed by a modulus of the tick counter is killed.) +; No known code (incl. Linux) tries to rely on the absolute position of +; unwired TLB entries after a TLBWR instruction. So this gives faster lookup +; when iterating over TLB, as the newer unwired entries will aggregate near +; the base of the table. Iron MIPSen do not iterate, they look up in parallel, +; ergo the original MIPS designer did not see any reason to attempt to order +; TLB entries by frequency of use. +;----------------------------------------------------------------------------- +%define TLBWR_CHEAT 1 +;----------------------------------------------------------------------------- + +;----------------------------------------------------------------------------- ; Alignment Grain ;----------------------------------------------------------------------------- %define GRAIN 32 diff -uNr a/m/mips_cpu.asm b/m/mips_cpu.asm --- a/m/mips_cpu.asm 93767fbf28ac3ca8a8d7cea92c2295d7f4a041c9b67b003edc7ca1a32ae624b423f8f33e5d099403a9e68e794f0f5d90f4f3f2a12e8d3cee4fcafb12e7502684 +++ b/m/mips_cpu.asm 35a5f7d843a515a6301c0d09957d3529f10f0443a50bd54177bcaaecc96054c502d2e14ccf1d5f106247dd2f566839ab49883e97b86cff1d5ad889652e8f5eaf @@ -40,12 +40,8 @@ SetIRQ TIMER_IRQ ; Timer reached limit, invoke timer IRQ _cycle_no_mips_timer: - - ;; Test if Interrupts are Disabled: - bt CP0_Status, CP0St_IE ; CF := CP0St_IE - jnc _cycle_no_irq ; If 0, IRQs are disabled, go to no_irq - test CP0_Status, (1 << CP0St_ERL) | (1 << CP0St_EXL) ; If ERL/EXL: - jnz _cycle_no_irq ; ... then also interrupts are disabled + + JMP_IF_IRQS_OFF _cycle_no_irq ; Skip IRQ processing if IRQs disabled GetSlaveIRQ ; See if slave threw interrupt @@ -64,7 +60,7 @@ ;; Copy InDelaySlot Flag to RunningDelaySlot Flag: Flg_Cpy RunningDelaySlot, InDelaySlot - jmp _Handle_Exception ; Handle exception and end cycle. + jmp _Handle_Exception_IRQ ; Handle IRQ exception and end cycle. _cycle_no_irq: ; If ints disabled or none pending diff -uNr a/m/mips_exc.asm b/m/mips_exc.asm --- a/m/mips_exc.asm fd686b4adbdb138a08f6b8f95620fd0533b2bd68c5b5056d24100f28b552cb6e7af74b32bfd739f029e065d257f456024d223073da3f0f8168e6d98f75be053d +++ b/m/mips_exc.asm 2ac1b0acacd2cd01845dc0e9ad36bae84577b76ace6be6e079c7e11b96a22508301b7b0012b119164ec5c87ab196d180da0a4e787d3b7ab9e3ed3436d82beea0 @@ -37,49 +37,31 @@ %endmacro ;----------------------------------------------------------------------------- -;------------------------------ -; Exception (and IRQ) Handler | ;----------------------------------------------------------------------------- -_Handle_Exception: +; Exception Prologue +;----------------------------------------------------------------------------- +%macro EXC_HANDLER_PROLOGUE 0 Flg_Off InDelaySlot ; Clear the InDelaySlot Flag bt CP0_Status, CP0St_EXL ; CF := EXL - jc _H_Exc_Off_Is_180 ; If EXL is 1, Offset := 0x180; else: + jc _H_Exc_Off_Is_180 ; If EXL is 1, Offset := 0x180; else: ;; EXL is 0: mov Sr(CP0_Epc), PC ; CP0_Epc := PC Flg_Get RunningDelaySlot ; Are we running the delay slot? - jnc _H_Exc_Not_R_Delay ; If yes, then skip, else: -_H_Exc_R_Delay: ; We ARE running the delay slot: + jnc %%no_delay ; If yes, then skip, else: +%%yes_delay: ; We ARE running the delay slot: sub Sr(CP0_Epc), 0x4 ; CP0_Epc := CP0_Epc - 0x4 bts CP0_Cause, CP0Cau_BD ; Set BD Flag in CP0_Cause - jmp _H_Exc_Test_TLB ; Proceed to test for TLBism -_H_Exc_Not_R_Delay: ; We are NOT running the delay slot: + jmp %%exc_prologue_done ; Continue in _Handle_Exception +%%no_delay: ; We are NOT running the delay slot: btr CP0_Cause, CP0Cau_BD ; Clear BD Flag in CP0_Cause -_H_Exc_Test_TLB: - ;; Test whether this was a TLB Exception: - GetEXC eax ; EAX := the current exception code - cmp eax, EXC_TLBL ; was it EXC_TLBL ? - je __H_Exc_Was_TLB ; ... if yes, go to H_Exc_Was_TLB - cmp eax, EXC_TLBS ; was it EXC_TLBS ? - je __H_Exc_Was_TLB ; ... if yes, go to H_Exc_Was_TLB - ;; This was NOT a TLB Exception: - cmp eax, EXC_Int ; was code EXC_Int ? - jne _H_Exc_Off_Is_180 ; if not, Offset := 0x180 - bt CP0_Cause, CP0Cau_IV ; Was CP0_Cause bit 23 (IV) zero? - jnc _H_Exc_Off_Is_180 ; if was zero, Offset := 0x180 - ;; If CP0_Cause bit 23 != 0: Offset := 0x200 - mov eax, 0x200 ; Offset := 0x200 - jmp _H_Exc_Have_Offset ; Go to H_Exc_Have_Offset -__H_Exc_Was_TLB: ; This WAS a TLB Exception: - Flg_Get ExcWasTLBNoMatch ; CF := 'TLB No Match' - ;; TODO: in 'cmips', this case was reversed? why? - ;; For now, we'll do likewise: - jnc _H_Exc_Off_Is_180 ; ... if 0, Offset := 0x180 - ;; Else: Offset := 0x0 - xor eax, eax ; Clear EAX (Offset := 0) - jmp _H_Exc_Have_Offset ; Finish up -_H_Exc_Off_Is_180: ; Offset := 0x180 - mov eax, 0x180 ; Set the Offset -_H_Exc_Have_Offset: ; We finished calculating the Offset: +%%exc_prologue_done: +%endmacro +;----------------------------------------------------------------------------- + +;----------------------------------------------------------------------------- +; Exception Epilogue: +;----------------------------------------------------------------------------- +_Exception_Epilogue: bts CP0_Status, CP0St_EXL ; Set the EXL Flag mov PC, eax ; PC := Offset (eax) mov eax, 0x80000000 ; Base that will be used if BEV=0 @@ -90,3 +72,34 @@ ;; Done handling exception jmp _cycle ; Start next cycle. ;----------------------------------------------------------------------------- + +;------------------------------------------ +; Exception Handler: TLB NoMatch Fastpath | +;----------------------------------------------------------------------------- +_Handle_Exception_TLB_NoMatch: + EXC_HANDLER_PROLOGUE + xor eax, eax ; Clear EAX (Offset := 0) + jmp _Exception_Epilogue ; Finish up +;----------------------------------------------------------------------------- + +;------------------------------------------------------- +; Exception Handler: Not-IRQ, Not-TLB-NoMatch Fastpath | +;----------------------------------------------------------------------------- +_Handle_Exception_Other: + EXC_HANDLER_PROLOGUE +_H_Exc_Off_Is_180: + mov eax, 0x180 ; Set the Offset + jmp _Exception_Epilogue ; Finish up +;----------------------------------------------------------------------------- + +;---------------------------------- +; Exception Handler: IRQ Fastpath | +;----------------------------------------------------------------------------- +_Handle_Exception_IRQ: + EXC_HANDLER_PROLOGUE + bt CP0_Cause, CP0Cau_IV ; Was CP0_Cause bit 23 (IV) zero? + jnc _H_Exc_Off_Is_180 ; if was zero, Offset := 0x180 + ;; If CP0_Cause bit 23 != 0: Offset := 0x200 + mov eax, 0x200 ; Offset := 0x200 + jmp _Exception_Epilogue ; Finish up +;----------------------------------------------------------------------------- diff -uNr a/m/mipsinst/i_instrs.asm b/m/mipsinst/i_instrs.asm --- a/m/mipsinst/i_instrs.asm 5b25235b8644d82e985ed6cc354e5e6e0d94aaaf0d0b5a7b33d8a5586cf67a90f13885398e5c10b935c9ee991b1cc720ae9b832dff9961d6090cf5e17a0445b9 +++ b/m/mipsinst/i_instrs.asm 42fca47ac9e2cc3d6cee432168b103fc0955cd846dac338b966b81de7d427a6cd682a3177b86ba3441bcb859fc4a6219011060292a8a8f756c053e391550e018 @@ -171,7 +171,7 @@ align GRAIN, db 0x90 _bad: SetEXC EXC_RI ; Set the EXC_RI Exception - jmp _Handle_Exception ; Go straight to exception handler. + jmp _Handle_Exception_Other ; Go straight to exception handler. ;----------------------------------------------------------------------------- ;----------------------------------------------------------------------------- @@ -707,7 +707,7 @@ bt Imm, 0 ; Get low bit of vAddr jnc _i_lh_aligned_ok ; If zero, vAddr was properly aligned; SetEXC EXC_AdEL ; If vAddr was NOT properly aligned: - jmp _Handle_Exception ; Go straight to exception handler. + jmp _Handle_Exception_Other ; Go straight to exception handler. _i_lh_aligned_ok: ; Load is to proceed normally: mov ecx, Imm ; Save vAddr to ECX xor TMP, TMP ; Clear TMP, where we will put the HW @@ -830,7 +830,7 @@ bt Imm, 0 ; Get low bit of vAddr jnc _i_lhu_aligned_ok ; If zero, vAddr was properly aligned; SetEXC EXC_AdEL ; If vAddr was NOT properly aligned: - jmp _Handle_Exception ; Go straight to exception handler. + jmp _Handle_Exception_Other ; Go straight to exception handler. _i_lhu_aligned_ok: ; Load is to proceed normally: mov ecx, Imm ; Save vAddr to ECX xor TMP, TMP ; Clear TMP, where we will put the HW @@ -933,7 +933,7 @@ bt Imm, 0 ; Get low bit of vAddr jnc _i_sh_aligned_ok ; If zero, vAddr was properly aligned; SetEXC EXC_AdES ; If vAddr was NOT properly aligned: - jmp _Handle_Exception ; Go straight to exception handler. + jmp _Handle_Exception_Other ; Go straight to exception handler. _i_sh_aligned_ok: mov AUX, Imm ; Save Imm (vAddr) to AUX ;; rS (ecx) is not needed any more, can be reused diff -uNr a/m/mipsinst/m_instrs.asm b/m/mipsinst/m_instrs.asm --- a/m/mipsinst/m_instrs.asm 0b4157a311317d55241b7a9700e16b0952cc540285d4922ec2d6626c29c3a9c83de29d18068d9be2a721b801cded2d930a4cdecf4991c5c2eab707e5e987bde6 +++ b/m/mipsinst/m_instrs.asm 931b5fd9ac59730bbcb95e9a9d3dba41483bbe6b3fc204ad8194397191795bacf3ef76df5335f8f17b3479a007de3a808df640fca949a7802b183bc25e7fe0c3 @@ -92,6 +92,25 @@ _m_tlbwr: ; no fields mov ecx, Sr(CP0_Wired) ; ecx := CP0_Wired +%ifdef TLBWR_CHEAT ; 'Cheat' behaviour (see knobs.asm for rationale) : + mov AUX, ecx ; save this index in AUX, we will use + mov edx, TLB_ENTRIES_COUNT - 1 ; index of last entry + ; Move all TLB entries after the Wired entries down by one slot: +.tlbwr_slide: ; Start by replacing the last entry with the next-to-last: + cmp edx, AUX ; time to stop? + je .tlbr_slide_done ; ... then stop. + mov ecx, edx ; ecx := edx + dec ecx ; ecx := ecx - 1 (prev. TLB index) + mov eax, TLB_E(ecx) ; eax := current TLB entry + mov TLB_E(edx), eax ; move the current into the next + mov eax, TLB_PFN_E(ecx) ; eax := current PFN_EVEN entry + mov TLB_PFN_E(edx), eax ; move the current into the next + mov eax, TLB_PFN_O(ecx) ; eax := current PFN_ODD entry + mov TLB_PFN_O(edx), eax ; move the current into the next + dec edx ; move back by one + jmp .tlbwr_slide ; Continue the slide. +.tlbr_slide_done: ; Now we freed up the top-most non-wired slot in TLB table: +%else ; 'Traditional' behaviour per the MIPS Standard: mov ebx, TLB_ENTRIES_COUNT ; ebx := #tlbentries sub ebx, ecx ; ebx := #tlbentries - Wired mov edx, 0 ; edx (upper half of dividend) := 0 @@ -99,7 +118,8 @@ div ebx ; edx:eax / ebx add edx, ecx ; edx (remainder) := edx + wired mov AUX, edx ; make edx the index for tlb write - call _write_tlb_entry ; Write the indexed TLB entry. +%endif + call _write_tlb_entry ; Write the AUX-indexed TLB entry. jmp _end_cycle ;----------------------------------------------------------------------------- @@ -446,7 +466,12 @@ test ecx, ecx ; Sel != 0 ? jnz _mtc0_unknown ; ... then unknown; else: and ebx, ~0x1F00 ; T := T & ~0x1F00 - mov Sr(CP0_EntryHi), ebx ; CP0_EntryHi := T + cmp ebx, Sr(CP0_EntryHi) ; Find whether changing CP0_EntryHi + je .Not_Changed_EntryHi ; ... if not, skip; +.Changed_EntryHi: ; If we are changing CP0_EntryHi: + Invalidate_TLB_Cache ; Invalidate both R and W TLB Caches + mov Sr(CP0_EntryHi), ebx ; CP0_EntryHi := ebx +.Not_Changed_EntryHi: jmp _end_cycle ; Done ;----------------------------------------------------------------------------- _mtc0_r11: ; 0x0b diff -uNr a/m/mipsinst/r_instrs.asm b/m/mipsinst/r_instrs.asm --- a/m/mipsinst/r_instrs.asm 40ef74622647a11ac973984a9f831ad39bbec47eac267e3f62fb30e36058ac4b01074b97585e9b36e0b32deb7344ec4c5aa6585e9571baa2e0432e8d78deb254 +++ b/m/mipsinst/r_instrs.asm f729ffa37ed0e9c666a6cde198317d24a05ad91612b0408c22b30a2d9c179b61f757bd6b8820a87c109d22786fb376c63a6a1252402f47ecb92dbd18f8769a80 @@ -298,7 +298,7 @@ _r_syscall: ; no fields SetEXC EXC_SYS ; Set the EXC_SYS Exception - jmp _Handle_Exception ; Go straight to exception handler. + jmp _Handle_Exception_Other ; Go straight to exception handler. ;----------------------------------------------------------------------------- ;----------------------------------------------------------------------------- @@ -632,7 +632,7 @@ cmp TMP, R(rT) ; CMP(TMP, Regs[rT]) jne _r_teq_neql SetEXC EXC_Tr ; Spring the Trap Exception. - jmp _Handle_Exception ; Go straight to exception handler. + jmp _Handle_Exception_Other ; Go straight to exception handler. _r_teq_neql: jmp _end_cycle ;----------------------------------------------------------------------------- @@ -656,7 +656,7 @@ cmp TMP, R(rT) ; CMP(TMP, Regs[rT]) je _r_tne_eql SetEXC EXC_Tr ; Spring the Trap Exception. - jmp _Handle_Exception ; Go straight to exception handler. + jmp _Handle_Exception_Other ; Go straight to exception handler. _r_tne_eql: jmp _end_cycle ;----------------------------------------------------------------------------- diff -uNr a/m/os/linux.asm b/m/os/linux.asm --- a/m/os/linux.asm 63181e522486b260324663a2c7cb928e8110114503a0711ac596f35176cb774bca680f59c83b3723a3abc4eb57a3fa3d10a657ca6e6bc79a4a3706325279068b +++ b/m/os/linux.asm d4b9d027bed497801c62b87f65849afe1c0ad557e304fe581dbc3e3117b082d735599b13ca8093a2ac97d463fc0a5e77c0c52d682a42ef1b50917cd8fa36ad72 @@ -34,23 +34,31 @@ push r14 push r15 - ; lea rsp,[rsp-16*4] - ; movdqu [rsp+16*0], xmm0 - ; movdqu [rsp+16*1], xmm1 - ; movdqu [rsp+16*2], xmm2 - ; movdqu [rsp+16*3], xmm2 - + ;; Save TLB cache: + lea rsp,[rsp-16*8] + movdqu [rsp+16*0], xmm5 + movdqu [rsp+16*1], xmm6 + movdqu [rsp+16*2], xmm7 + movdqu [rsp+16*3], xmm8 + movdqu [rsp+16*4], xmm9 + movdqu [rsp+16*5], xmm10 + movdqu [rsp+16*6], xmm11 + movdqu [rsp+16*7], xmm12 %endmacro ;----------------------------------------------------------------------------- ;----------------------------------------------------------------------------- %macro POPA 0 - - ; movdqu xmm1,[rsp+16*3] - ; movdqu xmm1,[rsp+16*2] - ; movdqu xmm1,[rsp+16*1] - ; movdqu xmm0,[rsp+16*0] - ; lea rsp,[rsp+16*4] + ;; Restore TLB cache: + movdqu xmm5, [rsp+16*0] + movdqu xmm6, [rsp+16*1] + movdqu xmm7, [rsp+16*2] + movdqu xmm8, [rsp+16*3] + movdqu xmm9, [rsp+16*4] + movdqu xmm10, [rsp+16*5] + movdqu xmm11, [rsp+16*6] + movdqu xmm12, [rsp+16*7] + lea rsp,[rsp+16*8] pop r15 pop r14 diff -uNr a/m/ram.asm b/m/ram.asm --- a/m/ram.asm 6ad0018398efc679130db6e4abdf4082d644bcbb623ce66c49899053b0a032a27ff1de0702efecd0a5bf6718adfbdf9f1c38b7e2f1efae34cfaf0258e0731f8d +++ b/m/ram.asm 016c026dbe4230bd120c0fc4269e61bd8a44b82580289efc90fed0792b5893a5727e069191fbfb0e32c3c40d2700b4a39a5acb0be1fdbfc475274c344368626a @@ -85,44 +85,40 @@ ;----------------------------------------------------------------------------- ;----------------------------------------------------------------------------- -; Translate_Address : virtual address in eax; output (physical addr) in eax +; Mark both Read and Write TLB Caches as blown: +;----------------------------------------------------------------------------- +%macro Invalidate_TLB_Cache 0 + and Flag_Reg, ~((1 << TLB_Rd_Cache_Valid) | (1 << TLB_Wr_Cache_Valid)) +%endmacro +;----------------------------------------------------------------------------- + +;----------------------------------------------------------------------------- +; Virt2Phys Read : virtual address in eax; output (physical addr) in eax ;----------------------------------------------------------------------------- align GRAIN, db 0x90 -_Virt_To_Phys: +_Virt_To_Phys_Read: bt eax, 31 ; CF := 31st (top) bit of vAddr - jc _Above_7FFFFFFF ; If 31st bit = 1, kseg 0/1/2; else: + jc .Above_7FFFFFFF ; If 31st bit = 1, kseg 0/1/2; else: ;; 0x00000000 <= vAddr <= 0x7FFFFFFF (kuseg) : bt CP0_Status, CP0St_ERL ; CF := CP0St_ERL Flag - jnc _TLB_Lookup ; If ERL = 0: TLB Lookup required; else: - jmp _No_Tlb_Lookup ; pAddr is equal to vAddr, return. -_Above_7FFFFFFF: + jnc .TLB_Lookup ; If ERL = 0: TLB Lookup required; else: + ret ; pAddr is equal to vAddr, return. +.Above_7FFFFFFF: bt eax, 30 ; CF := 30th (2nd from top) bt of vAddr - jc _Above_BFFFFFFF ; If 30th bit = 1 : kseg2; else: + jc .Above_BFFFFFFF ; If 30th bit = 1 : kseg2; else: ;; 0x80000000 <= vAddr <= 0x9FFFFFFF (kseg0) : ;; 0xA0000000 <= vAddr <= 0xBFFFFFFF (kseg1) : and eax, 0x1FFFFFFF ; kseg0 and kseg1: clear top 3 bits, - jmp _No_Tlb_Lookup ; i.e. pAddr := bottom 29 bts of vAddr. -_Above_BFFFFFFF: ;; 0xC0000000 <= vAddr <= 0xFFFFFFFF (kseg2) : - bt CP0_Status, CP0St_UM ; CF := CP0St_UM Flag - jnc _TLB_Lookup ; If UM = 0: K. Mode, so do TLB; else: - test CP0_Status, (1 << CP0St_EXL) | (1 << CP0St_ERL) ; EXL or ERL - jnz _TLB_Lookup ; If EXL && ERL, K. Mode, do TLB + ret ; i.e. pAddr := bottom 29 bts of vAddr. +.Above_BFFFFFFF: ;; 0xC0000000 <= vAddr <= 0xFFFFFFFF (kseg2) : + JMP_IF_KERNELMODE .TLB_Lookup ; If Kernel Mode, do TLB lookup; ;; Else: vAddr is in kseg2, but we are NOT in Kernel Mode: - Flg_Get IsWriting ; Is Writing? - jc _V2P_Eggog_Wr ; If so, we want to set AdES; -_V2P_Eggog_Rd: ; ... otherwise, set AdEL. SetEXC EXC_AdEL ; Fetch address error. - jmp _V2P_Eggog_Fin ; Proceed to abort. -_V2P_Eggog_Wr: - SetEXC EXC_AdES ; Store address error. -_V2P_Eggog_Fin: ;; Will go into exception handler instead of back to _Virt_xxx etc add rsp, 16 ; squelch return to _Virt_xxx and its caller - push _Handle_Exception ; 'return' directly to exc handler. -_No_Tlb_Lookup: - Flg_Off IsWriting ; Consume 'is writing' flag. + push _Handle_Exception_Other ; 'return' directly to exc handler. ret ; Done. -_TLB_Lookup: ; TLB Lookup Required: +.TLB_Lookup: ; TLB Lookup Required: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Save ebx, ecx, edx, AUX, to xmm ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -131,71 +127,235 @@ movd xmm2, edx movd xmm3, AUX ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - Flg_Off ExcWasTLBNoMatch ; Clear the ExcWasTLBNoMatch Flag + mov ecx, eax ; ecx := eax (vAddr) + and ecx, 0xFFFFF000 ; ecx := ecx & 0xFFFFF000 + shr ecx, 13 ; ecx := ecx >> 13 (get vAddr's Tag) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Find out whether we actually must do the lookup, or can use cached: + Flg_Get TLB_Rd_Cache_Valid ; Is Read TLB Cache valid? + jnc .Lookup_Must ; If Read TLB Cache invalid -- must! + bt eax, 12 ; Test odd/even junior bit + jc .Rd_Cached_Odd ; If odd, look at last Odd vAddr Tag +.Rd_Cached_Even: ; If even, look at last Even vAddr Tag + movd edx, Rd_E_Last_Tag ; edx := last Even vAddr's Tag + cmp ecx, edx ; is the current vAddr's Tag equal? + jne .Lookup_Must ; ... if not, must do the lookup dance; + ;; ... Otherwise, we have an Even cache hit: + movd ebx, Rd_E_Last_PFN ; ebx := last good Even PFN + jmp .Cache_Hit ; apply the PFN and wrap up. +.Rd_Cached_Odd: + movd edx, Rd_O_Last_Tag ; edx := last Odd vAddr's Tag + cmp ecx, edx ; is the current vAddr's Tag equal? + jne .Lookup_Must ; ... if not, must do the lookup dance; + ;; ... Otherwise, we have an Odd cache hit: + movd ebx, Rd_O_Last_PFN ; ebx := last good Odd PFN + jmp .Cache_Hit ; apply the PFN and wrap up. + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Failing the above, we must actually walk the TLB: +.Lookup_Must: + movd xmm4, ecx ; xmm4 := current vAddr's Tag ;; Get the active ASID: mov edx, Sr(CP0_EntryHi) ; edx := CP0_EntryHi and edx, 0xFF ; edx := edx & 0xFF (get current ASID) ;; For each slot in table (0 .. 15), attempt lookup xor AUX, AUX ; Start with the 0-th entry in table -_Lookup_TLB_E: - mov ecx, eax ; ecx := eax (vAddr) - and ecx, 0xFFFFF000 ; ecx := ecx & 0xFFFFF000 - shr ecx, 13 ; ecx := ecx >> 13 (get vAddr's Tag) +.Lookup_TLB_E: + movd ecx, xmm4 ; ecx := current vAddr's Tag mov ebx, TLB_E(AUX) ; ebx := current TLB entry and ebx, TLB_VPN2_Mask ; get VPN2 of this entry cmp ebx, ecx ; cmp(entry.VPN2, vAddr.tag) - jne _Lookup_TLB_E_Not_Here ; if entry.VPN2 != vAddr.tag: no match + jne .Lookup_TLB_E_Not_Here ; if entry.VPN2 != vAddr.tag: no match mov ebx, TLB_E(AUX) ; ebx := current TLB entry bt ebx, TLB_G ; is entry.G = 1? - jc _Lookup_TLB_E_Match ; then match. + jc .Lookup_TLB_E_Match ; then match. shr ebx, TLB_ASID_Shift ; ebx := ebx >> TLB_ASID_Shift and ebx, TLB_ASID_Mask ; ebx := entry.ASID cmp ebx, edx ; entry.ASID = current ASID ? - jne _Lookup_TLB_E_Not_Here ; if neither G=1 nor ASID match. + jne .Lookup_TLB_E_Not_Here ; if neither G=1 nor ASID match. mov ebx, TLB_E(AUX) ; ebx := current TLB entry -_Lookup_TLB_E_Match: ; TLB Match: +.Lookup_TLB_E_Match: ; TLB Match: bt eax, 12 ; Test odd/even junior bit - jc _Lookup_TLB_E_Match_Odd ; If odd: test V1, D1 -_Lookup_TLB_E_Match_Even: ; If even: test V0, D0 + jc .Lookup_TLB_E_Match_Odd ; If odd: test V1, D1 +.Lookup_TLB_E_Match_Even: ; If even: test V0, D0 + bt ebx, TLB_V0 ; Is entry.V0=1 ? + jnc .Lookup_TLB_E_Invalid ; If not, TLBRET_INVALID lea ecx, TLB_PFN_E(AUX) ; prepare to load even PFN entry + mov ebx, dword [ecx] ; Actually load the current PFN entry + movd Rd_E_Last_PFN, ebx ; Save the current PFN as last Even + movd ecx, xmm4 ; ecx := the current Tag + movd Rd_E_Last_Tag, ecx ; Save the current Tag as last Even + jmp .Lookup_TLB_E_Match_Yes ; Since we're reading: go to Match Yes +.Lookup_TLB_E_Match_Odd: ; Odd bit: + bt ebx, TLB_V1 ; Is entry.V1=1 ? + jnc .Lookup_TLB_E_Invalid ; If not, TLBRET_INVALID + lea ecx, TLB_PFN_O(AUX) ; prepare to load odd PFN entry + mov ebx, dword [ecx] ; Actually load the current PFN entry + movd Rd_O_Last_PFN, ebx ; Save the current PFN as last Odd + movd ecx, xmm4 ; ecx := the current Tag + movd Rd_O_Last_Tag, ecx ; Save the current Tag as last Odd +.Lookup_TLB_E_Match_Yes: ; This is the 'success' case + Flg_On TLB_Rd_Cache_Valid + ; Upon next TLB lookup, if cache is valid, and Tag remains same + ; as before, we can use the same PFN as was obtained last time + ; for the respective 12th bit arity of the vAddr! +.Cache_Hit: + and eax, 0xFFF ; vAddr := vAddr & 0xFFF + or eax, ebx ; vAddr := vAddr | entry.PFN[lowbit] + jmp _Lookup_TLB_Done ; vAddr is now correct pAddr, done. +.Lookup_TLB_E_Not_Here: ; try next one in the table, if any + inc AUX ; index := index + 1 + cmp AUX, TLB_ENTRIES_COUNT ; see if still in range 0 .. n-1 + jb .Lookup_TLB_E ; if in range, go to next entry + ;; ... else: + add rsp, 16 ; squelch return to _Virt_xxx and its caller + push _Handle_Exception_TLB_NoMatch ; 'return' straight to handler. + jmp _Lookup_TLB_E_WriteExtr ; Wrap up +.Lookup_TLB_E_Invalid: + SetEXC EXC_TLBL ; Set the EXC_TLBL Exception + add rsp, 16 ; squelch return to _Virt_xxx and its caller + push _Handle_Exception_Other ; 'return' straight to handler. + jmp _Lookup_TLB_E_WriteExtr ; Go to the common epilogue. +;----------------------------------------------------------------------------- + +;----------------------------------------------------------------------------- +; Virt2Phys Write : virtual address in eax; output (physical addr) in eax +;----------------------------------------------------------------------------- +align GRAIN, db 0x90 +_Virt_To_Phys_Write: + bt eax, 31 ; CF := 31st (top) bit of vAddr + jc .Above_7FFFFFFF ; If 31st bit = 1, kseg 0/1/2; else: + ;; 0x00000000 <= vAddr <= 0x7FFFFFFF (kuseg) : + bt CP0_Status, CP0St_ERL ; CF := CP0St_ERL Flag + jnc .TLB_Lookup ; If ERL = 0: TLB Lookup required; else: + ret ; pAddr is equal to vAddr, return. +.Above_7FFFFFFF: + bt eax, 30 ; CF := 30th (2nd from top) bt of vAddr + jc .Above_BFFFFFFF ; If 30th bit = 1 : kseg2; else: + ;; 0x80000000 <= vAddr <= 0x9FFFFFFF (kseg0) : + ;; 0xA0000000 <= vAddr <= 0xBFFFFFFF (kseg1) : + and eax, 0x1FFFFFFF ; kseg0 and kseg1: clear top 3 bits, + ret ; i.e. pAddr := bottom 29 bts of vAddr. +.Above_BFFFFFFF: ;; 0xC0000000 <= vAddr <= 0xFFFFFFFF (kseg2) : + JMP_IF_KERNELMODE .TLB_Lookup ; If Kernel Mode, do TLB lookup; + ;; Else: vAddr is in kseg2, but we are NOT in Kernel Mode: + SetEXC EXC_AdES ; Store address error. + ;; Will go into exception handler instead of back to _Virt_xxx etc + add rsp, 16 ; squelch return to _Virt_xxx and its caller + push _Handle_Exception_Other ; 'return' directly to exc handler. + ret ; Done. +.TLB_Lookup: ; TLB Lookup Required: + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Save ebx, ecx, edx, AUX, to xmm ;; + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + movd xmm0, ebx + movd xmm1, ecx + movd xmm2, edx + movd xmm3, AUX + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + mov ecx, eax ; ecx := eax (vAddr) + and ecx, 0xFFFFF000 ; ecx := ecx & 0xFFFFF000 + shr ecx, 13 ; ecx := ecx >> 13 (get vAddr's Tag) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Find out whether we actually must do the lookup, or can use cached: + Flg_Get TLB_Wr_Cache_Valid ; Is Write TLB Cache valid? + jnc .Lookup_Must ; If Write TLB Cache invalid -- must! + bt eax, 12 ; Test odd/even junior bit + jc .Wr_Cached_Odd ; If odd, look at last Odd vAddr Tag +.Wr_Cached_Even: ; If even, look at last Even vAddr Tag + movd edx, Wr_E_Last_Tag ; edx := last Even vAddr's Tag + cmp ecx, edx ; is the current vAddr's Tag equal? + jne .Lookup_Must ; ... if not, must do the lookup dance; + ;; ... Otherwise, we have an Even cache hit: + movd ebx, Wr_E_Last_PFN ; ebx := last good Even PFN + jmp .Cache_Hit ; apply the PFN and wrap up. +.Wr_Cached_Odd: + movd edx, Wr_O_Last_Tag ; edx := last Odd vAddr's Tag + cmp ecx, edx ; is the current vAddr's Tag equal? + jne .Lookup_Must ; ... if not, must do the lookup dance; + ;; ... Otherwise, we have an Odd cache hit: + movd ebx, Wr_O_Last_PFN ; ebx := last good Odd PFN + jmp .Cache_Hit ; apply the PFN and wrap up. + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Failing the above, we must actually walk the TLB: +.Lookup_Must: + movd xmm4, ecx ; xmm4 := current vAddr's Tag + ;; Get the active ASID: + mov edx, Sr(CP0_EntryHi) ; edx := CP0_EntryHi + and edx, 0xFF ; edx := edx & 0xFF (get current ASID) + ;; For each slot in table (0 .. 15), attempt lookup + xor AUX, AUX ; Start with the 0-th entry in table +.Lookup_TLB_E: + movd ecx, xmm4 ; ecx := current vAddr's Tag + mov ebx, TLB_E(AUX) ; ebx := current TLB entry + and ebx, TLB_VPN2_Mask ; get VPN2 of this entry + cmp ebx, ecx ; cmp(entry.VPN2, vAddr.tag) + jne .Lookup_TLB_E_Not_Here ; if entry.VPN2 != vAddr.tag: no match + mov ebx, TLB_E(AUX) ; ebx := current TLB entry + bt ebx, TLB_G ; is entry.G = 1? + jc .Lookup_TLB_E_Match ; then match. + shr ebx, TLB_ASID_Shift ; ebx := ebx >> TLB_ASID_Shift + and ebx, TLB_ASID_Mask ; ebx := entry.ASID + cmp ebx, edx ; entry.ASID = current ASID ? + jne .Lookup_TLB_E_Not_Here ; if neither G=1 nor ASID match. + mov ebx, TLB_E(AUX) ; ebx := current TLB entry +.Lookup_TLB_E_Match: ; TLB Match: + bt eax, 12 ; Test odd/even junior bit + jc .Lookup_TLB_E_Match_Odd ; If odd: test V1, D1 +.Lookup_TLB_E_Match_Even: ; If even: test V0, D0 bt ebx, TLB_V0 ; Is entry.V0=1 ? - jnc _Lookup_TLB_E_Invalid ; If not, TLBRET_INVALID + jnc .Lookup_TLB_E_Invalid ; If not, TLBRET_INVALID bt ebx, TLB_D0 ; Is entry.D0=1 ? - jc _Lookup_TLB_E_Match_Yes ; If entry.D0=1, then Match Yes - jmp _Lookup_TLB_E_Match_Wr ; else, go to 'is writing?' -_Lookup_TLB_E_Match_Odd: ; Odd bit: - lea ecx, TLB_PFN_O(AUX) ; prepare to load odd PFN entry + jnc .Lookup_TLB_E_Dirty ; If not, go to 'Dirty' + ;; Not invalid or dirty: + lea ecx, TLB_PFN_E(AUX) ; prepare to load even PFN entry + mov ebx, dword [ecx] ; Actually load the current PFN entry + movd Wr_E_Last_PFN, ebx ; Save the current PFN as last Even + movd ecx, xmm4 ; ecx := the current Tag + movd Wr_E_Last_Tag, ecx ; Save the current Tag as last Even + jmp .Lookup_TLB_E_Match_Yes ; ;; Proceed to 'match' : +.Lookup_TLB_E_Match_Odd: ; Odd bit: bt ebx, TLB_V1 ; Is entry.V1=1 ? - jnc _Lookup_TLB_E_Invalid ; If not, TLBRET_INVALID + jnc .Lookup_TLB_E_Invalid ; If not, TLBRET_INVALID bt ebx, TLB_D1 ; Is entry.D1=1 ? - jc _Lookup_TLB_E_Match_Yes ; If entry.D1=1, then Match Yes -_Lookup_TLB_E_Match_Wr: - Flg_Get IsWriting ; Is Writing? - jnc _Lookup_TLB_E_Match_Yes ; If not writing, go to Match Yes -_Lookup_TLB_E_Dirty: ; ... else, Dirty: - SetEXC EXC_Mod ; Set the EXC_Mod Exception - jmp _Lookup_TLB_E_WriteExtr ; Write the 'extra data' and finish. -_Lookup_TLB_E_Match_Yes: ; This is the 'success' case + jnc .Lookup_TLB_E_Dirty ; If not, go to 'Dirty' + ;; Not invalid or dirty: + lea ecx, TLB_PFN_O(AUX) ; prepare to load odd PFN entry mov ebx, dword [ecx] ; Actually load the current PFN entry + movd Wr_O_Last_PFN, ebx ; Save the current PFN as last Odd + movd ecx, xmm4 ; ecx := the current Tag + movd Wr_O_Last_Tag, ecx ; Save the current Tag as last Odd + ;; Proceed to 'match' : +.Lookup_TLB_E_Match_Yes: ; This is the 'success' case + Flg_On TLB_Wr_Cache_Valid + ; Upon next TLB lookup, if cache is valid, and Tag remains same + ; as before, we can use the same PFN as was obtained last time + ; for the respective 12th bit arity of the vAddr! +.Cache_Hit: and eax, 0xFFF ; vAddr := vAddr & 0xFFF or eax, ebx ; vAddr := vAddr | entry.PFN[lowbit] jmp _Lookup_TLB_Done ; vAddr is now correct pAddr, done. -_Lookup_TLB_E_Not_Here: ; try next one in the table, if any +.Lookup_TLB_E_Not_Here: ; try next one in the table, if any inc AUX ; index := index + 1 cmp AUX, TLB_ENTRIES_COUNT ; see if still in range 0 .. n-1 - jb _Lookup_TLB_E ; if in range, go to next entry + jb .Lookup_TLB_E ; if in range, go to next entry ;; ... else: - Flg_On ExcWasTLBNoMatch ; Set the ExcWasTLBNoMatch Flag - ;; ... now drop down into 'invalid' : -_Lookup_TLB_E_Invalid: - Flg_Get IsWriting ; Was Writing? - jc _Lookup_TLB_E_Invalid_W ; If so, we want to set EXC_TLBS -_Lookup_TLB_E_Invalid_R: ; Otherwise, set EXC_TLBL exception - SetEXC EXC_TLBL ; Set the EXC_TLBL Exception + add rsp, 16 ; squelch return to _Virt_xxx and its caller + push _Handle_Exception_TLB_NoMatch ; 'return' straight to handler. + jmp _Lookup_TLB_E_WriteExtr ; Wrap up +.Lookup_TLB_E_Dirty: ; ... else, Dirty: + SetEXC EXC_Mod ; Set the EXC_Mod Exception + add rsp, 16 ; squelch return to _Virt_xxx and its caller + push _Handle_Exception_Other ; 'return' straight to handler. jmp _Lookup_TLB_E_WriteExtr ; Write the 'extra data' and finish. -_Lookup_TLB_E_Invalid_W: +.Lookup_TLB_E_Invalid: ; Invalid Write: SetEXC EXC_TLBS ; Set the EXC_TLBS Exception - ;; then drop down to 'write extra' : + add rsp, 16 ; squelch return to _Virt_xxx and its caller + push _Handle_Exception_Other ; 'return' straight to handler. + ;; then drop down to _Lookup_TLB_E_WriteExtr + +;----------------------------------------------------------------------------- +; Epilogue common to _Virt_To_Phys_Read and _Virt_To_Phys_Write: +;----------------------------------------------------------------------------- _Lookup_TLB_E_WriteExtr: ; Write the 'extra data' and finish mov Sr(CP0_BadVAddr), eax ; CP0_BadVAddr := vAddr mov ecx, eax ; ecx := vAddr @@ -210,13 +370,15 @@ and ebx, 0xFF ; ebx := ebx & 0xFF and ecx, 0xFFFFE000 ; ecx := ecx & 0xFFFFE000 or ebx, ecx ; ebx := ebx | ecx + cmp ebx, Sr(CP0_EntryHi) ; Find whether changing CP0_EntryHi + je .Not_Changed_EntryHi ; ... if not, skip; +.Changed_EntryHi: ; If we are changing CP0_EntryHi: + Invalidate_TLB_Cache ; Invalidate both R and W TLB Caches mov Sr(CP0_EntryHi), ebx ; CP0_EntryHi := ebx +.Not_Changed_EntryHi: ;; Will go into exception handler instead of back to _Virt_xxx etc - add rsp, 16 ; squelch return to _Virt_xxx and its caller - push _Handle_Exception ; 'return' directly to exc handler. ;; and drop into 'done' : _Lookup_TLB_Done: - Flg_Off IsWriting ; Consume 'is writing' flag. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Restore ebx, ecx, edx, AUX, from xmm ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -234,7 +396,7 @@ ; 2) would go faster if used tmp instead of self-clobbering rax ? align GRAIN, db 0x90 _Virt_Read_Word: - call _Virt_To_Phys ; Transform vAddr to pAddr + call _Virt_To_Phys_Read ; Transform vAddr to pAddr (for Read) test eax, 0x3 ; Are any of the bottom 2 bits set? jnz _V_Rd_Word_Unaligned ; If yes, go to eggog. Else: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -256,7 +418,7 @@ SetEXC EXC_AdEL ; Fetch address error. ;; Will go into exception handler instead of back to caller add rsp, 8 ; squelch return to original caller - push _Handle_Exception ; 'return' directly to exc handler. + push _Handle_Exception_Other ; 'return' directly to exc handler. ret ; Go there. ;----------------------------------------------------------------------------- @@ -268,8 +430,7 @@ ; 3) do we need to explicitly zero-extend rax here? align GRAIN, db 0x90 _Virt_Write_Word: - Flg_On IsWriting ; Tell the translator that we're writing - call _Virt_To_Phys ; Transform vAddr to pAddr + call _Virt_To_Phys_Write ; Transform vAddr to pAddr (for Write) test eax, 0x3 ; Are any of the bottom 2 bits set? jnz _V_Wr_Word_Unaligned ; If yes, go to eggog. Else: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -291,7 +452,7 @@ SetEXC EXC_AdES ; Store address error. ;; Will go into exception handler instead of back to caller add rsp, 8 ; squelch return to original caller - push _Handle_Exception ; 'return' directly to exc handler. + push _Handle_Exception_Other ; 'return' directly to exc handler. ret ; Go there. ;----------------------------------------------------------------------------- @@ -300,7 +461,7 @@ ;----------------------------------------------------------------------------- align GRAIN, db 0x90 _Virt_Read_Byte: - call _Virt_To_Phys ; Transform vAddr to pAddr + call _Virt_To_Phys_Read ; Transform vAddr to pAddr (for Read) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; If pAddr is in Memory-Mapped Device space: JMP_If_In eax, MMIO_BASE, MMIO_TOP, _Phys_Device_Read_Byte @@ -324,7 +485,7 @@ SetEXC EXC_AdEL ; Fetch address error. ;; Will go into exception handler instead of back to caller add rsp, 8 ; squelch return to original caller - push _Handle_Exception ; 'return' directly to exc handler. + push _Handle_Exception_Other ; 'return' directly to exc handler. ret ; Go there. ;----------------------------------------------------------------------------- @@ -333,8 +494,7 @@ ;----------------------------------------------------------------------------- align GRAIN, db 0x90 _Virt_Write_Byte: - Flg_On IsWriting ; Tell the translator that we're writing - call _Virt_To_Phys ; Transform vAddr to pAddr + call _Virt_To_Phys_Write ; Transform vAddr to pAddr (for Write) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; If pAddr is in Memory-Mapped Device space: JMP_If_In eax, MMIO_BASE, MMIO_TOP, _Phys_Device_Write_Byte @@ -357,7 +517,7 @@ SetEXC EXC_AdES ; Store address error. ;; Will go into exception handler instead of back to caller add rsp, 8 ; squelch return to original caller - push _Handle_Exception ; 'return' directly to exc handler. + push _Handle_Exception_Other ; 'return' directly to exc handler. ret ; Go there. ;----------------------------------------------------------------------------- @@ -416,5 +576,6 @@ ;; Store PFN: mov TLB_PFN_E(AUX), edx ; Store PFN0 mov TLB_PFN_O(AUX), ebx ; Store PFN1 + Invalidate_TLB_Cache ; Invalidate both R and W TLB Caches ret ; Done. ;-----------------------------------------------------------------------------