Seperate gmmu pagetable

.vscode/launch.json

@@ -166,7 +166,7 @@
             "program": "${workspaceFolder}/samples/fir",
             "args": [
                 "-timing",
-                "-length=8192",
+                "-length=81920",
                 "-report-all",
             ],
         },

driver/builder.go

@@ -17,6 +17,7 @@
 	useMagicMemoryCopy  bool
 	middlewareD2HCycles int
 	middlewareH2DCycles int
+	cpuMemorySize       uint64
 }
 
 // MakeBuilder creates a driver builder with some default configuration
@@ -74,6 +75,12 @@
 	return b
 }
 
+// WithMemorySize sets the memory size of the CPU.
+func (b Builder) WithCPUMemorySize(memorySize uint64) Builder {
+	b.memorySize = memorySize
+	return b
+}
+
 // Build creates a driver.
 func (b Builder) Build(name string) *Driver {
 	driver := new(Driver)
@@ -125,7 +132,7 @@
 		Type:     internal.DeviceTypeCPU,
 		MemState: internal.NewDeviceMemoryState(d.Log2PageSize),
 	}
-	cpu.SetTotalMemSize(4 * mem.GB)
+	cpu.SetTotalMemSize(b.memorySize)
 
 	d.memAllocator.RegisterDevice(cpu)
 	d.devices = append(d.devices, cpu)

driver/driver.go

@@ -142,6 +142,7 @@ type DeviceProperties struct {
 func (d *Driver) RegisterGPU(
 	commandProcessorPort sim.Port,
 	properties DeviceProperties,
+	gmmuPageTable vm.PageTable,
 ) {
 	d.GPUs = append(d.GPUs, commandProcessorPort)
 
@@ -156,8 +157,13 @@ func (d *Driver) RegisterGPU(
 	}
 	gpuDevice.SetTotalMemSize(properties.DRAMSize)
 	d.memAllocator.RegisterDevice(gpuDevice)
-
 	d.devices = append(d.devices, gpuDevice)
+
+	for _, page := range d.memAllocator.GetVAddrToPageMapping() {
+		if page.DeviceID == uint64(gpuDevice.ID) {
+			gmmuPageTable.Insert(page)
+		}
+	}
 }
 
 // Tick ticks

driver/internal/memoryallocator.go

@@ -22,6 +22,8 @@ type MemoryAllocator interface {
 		vAddr uint64,
 		unified bool,
 	) vm.Page
+
+	GetVAddrToPageMapping() map[uint64]vm.Page
 }
 
 // NewMemoryAllocator creates a new memory allocator.
@@ -63,9 +65,7 @@ func (a *memoryAllocatorImpl) RegisterDevice(device *Device) {
 
 	state := device.MemState
 	state.setInitialAddress(a.totalStorageByteSize)
-
 	a.totalStorageByteSize += state.getStorageSize()
-
 	a.devices[device.ID] = device
 }
 
@@ -285,3 +285,13 @@ func (a *memoryAllocatorImpl) Free(ptr uint64) {
 
 	a.removePage(ptr)
 }
+
+func (a *memoryAllocatorImpl) GetVAddrToPageMapping() map[uint64]vm.Page {
+	a.Lock()
+	defer a.Unlock()
+	copy := make(map[uint64]vm.Page, len(a.vAddrToPageMapping))
+	for vAddr, page := range a.vAddrToPageMapping {
+		copy[vAddr] = page
+	}
+	return copy
+}

samples/runner/emuplatform.go

@@ -98,7 +98,8 @@ func (b EmuBuilder) Build() *Platform {
 		gpuDriver.RegisterGPU(cpPort, driver.DeviceProperties{
 			DRAMSize: 4 * mem.GB,
 			CUCount:  64,
-		})
+		},
+			gpuBuilder.pageTable)
 		connection.PlugIn(cpPort, 64)
 
 		b.gpus = append(b.gpus, gpu)

samples/runner/flag.go

@@ -131,6 +131,13 @@ func (r *Runner) ParseFlag() *Runner {
 		r.ReportSIMDBusyTime = true
 		r.ReportDRAMTransactionCount = true
 		r.ReportRDMATransactionCount = true
+		r.ReportGMMULatency = true
+		r.ReportMMULatency = true
+		r.ReportGMMUTransactionCount = true
+		r.ReportMMUTransactionCount = true
+		r.ReportSIMDBusyTime = true
+		r.ReportGMMUCacheHitRate = true
+		r.ReportGMMUCacheLatency = true
 		r.ReportCPIStack = true
 	}
 

samples/runner/platform.go

@@ -1,6 +1,8 @@
 package runner
 
 import (
+	"github.com/sarchlab/akita/v3/mem/vm/gmmu"
+	"github.com/sarchlab/akita/v3/mem/vm/mmu"
 	"github.com/sarchlab/akita/v3/sim"
 	"github.com/sarchlab/akita/v3/tracing"
 	"github.com/sarchlab/mgpusim/v3/driver"
@@ -27,16 +29,20 @@
 	Domain           *sim.Domain
 	CommandProcessor *cp.CommandProcessor
 	RDMAEngine       *rdma.Comp
-	PMC              *pagemigrationcontroller.PageMigrationController
-	CUs              []TraceableComponent
-	SIMDs            []TraceableComponent
-	L1VCaches        []TraceableComponent
-	L1SCaches        []TraceableComponent
-	L1ICaches        []TraceableComponent
-	L2Caches         []TraceableComponent
-	L1VTLBs          []TraceableComponent
-	L1STLBs          []TraceableComponent
-	L1ITLBs          []TraceableComponent
-	L2TLBs           []TraceableComponent
-	MemControllers   []TraceableComponent
+	MMUEngine        *mmu.MMU
+	GMMUEngine       *gmmu.GMMU
+
+	PMC            *pagemigrationcontroller.PageMigrationController
+	CUs            []TraceableComponent
+	SIMDs          []TraceableComponent
+	L1VCaches      []TraceableComponent
+	L1SCaches      []TraceableComponent
+	L1ICaches      []TraceableComponent
+	L2Caches       []TraceableComponent
+	L1VTLBs        []TraceableComponent
+	L1STLBs        []TraceableComponent
+	L1ITLBs        []TraceableComponent
+	L2TLBs         []TraceableComponent
+	GMMUCache      []TraceableComponent
+	MemControllers []TraceableComponent
 }

samples/runner/r9nanobuilder.go

@@ -11,7 +11,9 @@ import (
 	"github.com/sarchlab/akita/v3/mem/cache/writethrough"
 	"github.com/sarchlab/akita/v3/mem/dram"
 	"github.com/sarchlab/akita/v3/mem/mem"
+	"github.com/sarchlab/akita/v3/mem/vm"
 	"github.com/sarchlab/akita/v3/mem/vm/addresstranslator"
+	"github.com/sarchlab/akita/v3/mem/vm/gmmu"
 	"github.com/sarchlab/akita/v3/mem/vm/mmu"
 	"github.com/sarchlab/akita/v3/mem/vm/tlb"
 	"github.com/sarchlab/akita/v3/monitoring"
@@ -65,6 +67,8 @@ type R9NanoGPUBuilder struct {
 	l1sTLBs                 []*tlb.TLB
 	l1iTLBs                 []*tlb.TLB
 	l2TLBs                  []*tlb.TLB
+	gmmuCache               *tlb.TLB
+	gmmu                    *gmmu.GMMU
 	drams                   []*dram.MemController
 	lowModuleFinderForL1    *mem.InterleavedLowModuleFinder
 	lowModuleFinderForL2    *mem.InterleavedLowModuleFinder
@@ -73,6 +77,7 @@ type R9NanoGPUBuilder struct {
 	rdmaEngine              *rdma.Comp
 	pageMigrationController *pagemigrationcontroller.PageMigrationController
 	globalStorage           *mem.Storage
+	pageTable               vm.PageTable
 
 	internalConn           *sim.DirectConnection
 	l1TLBToL2TLBConnection *sim.DirectConnection
@@ -225,19 +230,31 @@ func (b R9NanoGPUBuilder) WithGlobalStorage(
 	return b
 }
 
+// WithGMMUPageTable lets GPU to initialize it's page table.
+func (b R9NanoGPUBuilder) WithGMMUPageTable(
+	pageTable vm.PageTable,
+) R9NanoGPUBuilder {
+	b.pageTable = pageTable
+	return b
+}
+
 // Build creates a pre-configure GPU similar to the AMD R9 Nano GPU.
 func (b R9NanoGPUBuilder) Build(name string, id uint64) *GPU {
 	b.createGPU(name, id)
 	b.buildSAs()
 	b.buildL2Caches()
 	b.buildDRAMControllers()
 	b.buildCP()
+	b.buildGMMU()
+	b.buildGMMUCache()
 	b.buildL2TLB()
 
 	b.connectCP()
 	b.connectL2AndDRAM()
 	b.connectL1ToL2()
 	b.connectL1TLBToL2TLB()
+	b.connectL2TLBToGMMUCache()
+	b.connectGMMUCachetoGMMU()
 
 	b.populateExternalPorts()
 
@@ -254,6 +271,7 @@ func (b *R9NanoGPUBuilder) populateExternalPorts() {
 		name := fmt.Sprintf("Translation_%02d", i)
 		b.gpu.Domain.AddPort(name, l2TLB.GetPortByName("Bottom"))
 	}
+	b.gpu.Domain.AddPort("GMMU", b.gmmu.GetPortByName("Bottom"))
 }
 
 func (b *R9NanoGPUBuilder) createGPU(name string, id uint64) {
@@ -540,6 +558,67 @@ func (b *R9NanoGPUBuilder) buildL2Caches() {
 		if b.monitor != nil {
 			b.monitor.RegisterComponent(l2)
 		}
+		if b.perfAnalyzer != nil {
+			b.perfAnalyzer.RegisterComponent(l2)
+		}
+	}
+}
+
+func (b *R9NanoGPUBuilder) buildGMMUCache() {
+	// numWays := 128
+	// test:= int(b.dramSize / (1 << b.log2PageSize) / uint64(numWays))
+	builder := tlb.MakeBuilder().
+		WithEngine(b.engine).
+		WithFreq(b.freq).
+		WithNumWays(8).
+		WithNumSets(16).
+		WithNumMSHREntry(32).
+		WithNumReqPerCycle(32).
+		WithPageSize(1 << b.log2PageSize).
+		WithLowModule(b.gmmu.GetPortByName("Top"))
+
+	gmmuCache := builder.Build(fmt.Sprintf("%s.GMMUCache", b.gpuName))
+	b.gmmuCache = gmmuCache
+	b.gpu.GMMUCache = append(b.gpu.GMMUCache, gmmuCache)
+	// b.gpu.L2TLBs = append(b.gpu.L2TLBs, l2TLB)
+
+	if b.enableVisTracing {
+		tracing.CollectTrace(b.gmmuCache, b.visTracer)
+	}
+
+	if b.monitor != nil {
+		b.monitor.RegisterComponent(b.gmmuCache)
+	}
+
+	if b.perfAnalyzer != nil {
+		b.perfAnalyzer.RegisterComponent(b.gmmuCache)
+	}
+}
+
+func (b *R9NanoGPUBuilder) buildGMMU() {
+	gmmu := gmmu.MakeBuilder().
+		WithEngine(b.engine).
+		WithFreq(b.freq).
+		WithDeviceID(b.gpuID).
+		WithLog2PageSize(b.log2PageSize).
+		WithMaxNumReqInFlight(8).
+		WithPageTable(b.pageTable).WithPageWalkingLatency(100).
+		WithLowModule(b.mmu.GetPortByName("Top")).
+		Build(fmt.Sprintf("%s.GMMU", b.gpuName))
+
+	b.gmmu = gmmu
+	b.gpu.GMMUEngine = b.gmmu
+
+	if b.enableVisTracing {
+		tracing.CollectTrace(b.gmmu, b.visTracer)
+	}
+
+	if b.monitor != nil {
+		b.monitor.RegisterComponent(b.gmmu)
+	}
+
+	if b.perfAnalyzer != nil {
+		b.perfAnalyzer.RegisterComponent(b.gmmu)
 	}
 }
 
@@ -824,7 +903,7 @@ func (b *R9NanoGPUBuilder) buildL2TLB() {
 		WithNumMSHREntry(64).
 		WithNumReqPerCycle(1024).
 		WithPageSize(1 << b.log2PageSize).
-		WithLowModule(b.mmu.GetPortByName("Top"))
+		WithLowModule(b.gmmuCache.GetPortByName("Top"))
 
 	l2TLB := builder.Build(fmt.Sprintf("%s.L2TLB", b.gpuName))
 	b.l2TLBs = append(b.l2TLBs, l2TLB)
@@ -854,3 +933,24 @@ func (b *R9NanoGPUBuilder) connectWithDirectConnection(
 	conn.PlugIn(port1, bufferSize)
 	conn.PlugIn(port2, bufferSize)
 }
+
+func (b *R9NanoGPUBuilder) connectL2TLBToGMMUCache() {
+	conn := sim.NewDirectConnection(
+		b.gpuName+".L2TLBtoGMMUCache",
+		b.engine, b.freq,
+	)
+	conn.PlugIn(b.gmmuCache.GetPortByName("Top"), 64)
+
+	for _, l2TLB := range b.l2TLBs {
+		conn.PlugIn(l2TLB.GetPortByName("Bottom"), 64)
+	}
+}
+
+func (b *R9NanoGPUBuilder) connectGMMUCachetoGMMU() {
+	conn := sim.NewDirectConnection(
+		b.gpuName+".GMMUCacheToGMMU",
+		b.engine, b.freq,
+	)
+	conn.PlugIn(b.gmmu.GetPortByName("Top"), 64)
+	conn.PlugIn(b.gmmuCache.GetPortByName("Bottom"), 64)
+}