Merge branch 'v3' into 49-perf-analyzer-cannot-be-used-in-mgpusim

sarchlab · Mar 27, 2024 · 4ea987b · 4ea987b
2 parents ad4cac6 + 2d87a0e
commit 4ea987b
Show file tree

Hide file tree

Showing 4 changed files with 73 additions and 13 deletions.
diff --git a/driver/builder.go b/driver/builder.go
@@ -9,12 +9,14 @@ import (
 
 // A Builder can build a driver.
 type Builder struct {
-	engine             sim.Engine
-	freq               sim.Freq
-	log2PageSize       uint64
-	pageTable          vm.PageTable
-	globalStorage      *mem.Storage
-	useMagicMemoryCopy bool
+	engine              sim.Engine
+	freq                sim.Freq
+	log2PageSize        uint64
+	pageTable           vm.PageTable
+	globalStorage       *mem.Storage
+	useMagicMemoryCopy  bool
+	middlewareD2HCycles int
+	middlewareH2DCycles int
 }
 
 // MakeBuilder creates a driver builder with some default configuration
@@ -62,6 +64,16 @@ func (b Builder) WithMagicMemoryCopyMiddleware() Builder {
 	return b
 }
 
+func (b Builder) WithD2HCycles(d2hCycles int) Builder {
+	b.middlewareD2HCycles = d2hCycles
+	return b
+}
+
+func (b Builder) WithH2DCycles(h2dCycles int) Builder {
+	b.middlewareH2DCycles = h2dCycles
+	return b
+}
+
 // Build creates a driver.
 func (b Builder) Build(name string) *Driver {
 	driver := new(Driver)
@@ -87,7 +99,9 @@ func (b Builder) Build(name string) *Driver {
 		driver.middlewares = append(driver.middlewares, globalStorageMemoryCopyMiddleware)
 	} else {
 		defaultMemoryCopyMiddleware := &defaultMemoryCopyMiddleware{
-			driver: driver,
+			driver:       driver,
+			cyclesPerD2H: b.middlewareD2HCycles,
+			cyclesPerH2D: b.middlewareH2DCycles,
 		}
 		driver.middlewares = append(driver.middlewares, defaultMemoryCopyMiddleware)
 	}

diff --git a/driver/driver_test.go b/driver/driver_test.go
@@ -40,6 +40,8 @@ var _ = ginkgo.Describe("Driver", func() {
 			WithEngine(engine).
 			WithLog2PageSize(log2PageSize).
 			WithPageTable(pageTable).
+			WithD2HCycles(1).
+			WithH2DCycles(1).
 			Build("Driver")
 		driver.gpuPort = toGPUs
 		driver.mmuPort = toMMU
@@ -126,12 +128,20 @@ var _ = ginkgo.Describe("Driver", func() {
 				GetDeviceIDByPAddr(uint64(0x1_0000_2000)).
 				Return(1)
 
+			toGPUs.EXPECT().Peek().Return(nil).AnyTimes()
+			toGPUs.EXPECT().Peek().Return(nil).AnyTimes()
 			toGPUs.EXPECT().Peek().Return(nil).AnyTimes()
 			toMMU.EXPECT().Retrieve(sim.VTimeInSec(11)).Return(nil)
+			toMMU.EXPECT().Retrieve(sim.VTimeInSec(12)).Return(nil)
+			toMMU.EXPECT().Retrieve(sim.VTimeInSec(13)).Return(nil)
 
+			engine.EXPECT().Schedule(gomock.AssignableToTypeOf(sim.TickEvent{}))
+			engine.EXPECT().Schedule(gomock.AssignableToTypeOf(sim.TickEvent{}))
 			engine.EXPECT().Schedule(gomock.AssignableToTypeOf(sim.TickEvent{}))
 
 			driver.Handle(sim.MakeTickEvent(11, nil))
+			driver.Handle(sim.MakeTickEvent(12, nil))
+			driver.Handle(sim.MakeTickEvent(13, nil))
 
 			Expect(driver.requestsToSend).To(HaveLen(4))
 			Expect(cmdQueue.IsRunning).To(BeTrue())
@@ -226,13 +236,23 @@ var _ = ginkgo.Describe("Driver", func() {
 				GetDeviceIDByPAddr(uint64(0x1_0000_0100)).
 				Return(1)
 
+			toGPUs.EXPECT().Peek().Return(nil).AnyTimes()
+			toGPUs.EXPECT().Peek().Return(nil).AnyTimes()
 			toGPUs.EXPECT().Peek().Return(nil).AnyTimes()
 			toMMU.EXPECT().Retrieve(sim.VTimeInSec(11)).Return(nil)
+			toMMU.EXPECT().Retrieve(sim.VTimeInSec(12)).Return(nil)
+			toMMU.EXPECT().Retrieve(sim.VTimeInSec(13)).Return(nil)
 
+			engine.EXPECT().Schedule(
+				gomock.AssignableToTypeOf(sim.TickEvent{}))
+			engine.EXPECT().Schedule(
+				gomock.AssignableToTypeOf(sim.TickEvent{}))
 			engine.EXPECT().Schedule(
 				gomock.AssignableToTypeOf(sim.TickEvent{}))
 
 			driver.Handle(sim.MakeTickEvent(11, nil))
+			driver.Handle(sim.MakeTickEvent(12, nil))
+			driver.Handle(sim.MakeTickEvent(13, nil))
 
 			Expect(cmdQueue.IsRunning).To(BeTrue())
 			Expect(cmd.Reqs).To(HaveLen(1))

diff --git a/driver/memorycopy.go b/driver/memorycopy.go
@@ -12,6 +12,12 @@ import (
 // communication.
 type defaultMemoryCopyMiddleware struct {
 	driver *Driver
+
+	cyclesPerH2D int
+	cyclesPerD2H int
+	cyclesLeft   int
+
+	awaitingReqs []sim.Msg
 }
 
 func (m *defaultMemoryCopyMiddleware) ProcessCommand(
@@ -67,7 +73,8 @@ func (m *defaultMemoryCopyMiddleware) processMemCopyH2DCommand(
 			rawBytes[offset:offset+sizeToCopy],
 			pAddr)
 		cmd.Reqs = append(cmd.Reqs, req)
-		m.driver.requestsToSend = append(m.driver.requestsToSend, req)
+		m.awaitingReqs = append(m.awaitingReqs, req)
+		// m.driver.requestsToSend = append(m.driver.requestsToSend, req)
 
 		sizeLeft -= sizeToCopy
 		addr += sizeToCopy
@@ -76,6 +83,8 @@ func (m *defaultMemoryCopyMiddleware) processMemCopyH2DCommand(
 		m.driver.logTaskToGPUInitiate(now, cmd, req)
 	}
 
+	m.cyclesLeft = m.cyclesPerH2D
+
 	queue.IsRunning = true
 
 	return true
@@ -114,7 +123,8 @@ func (m *defaultMemoryCopyMiddleware) processMemCopyD2HCommand(
 			m.driver.gpuPort, m.driver.GPUs[gpuID-1],
 			pAddr, cmd.RawData[offset:offset+sizeToCopy])
 		cmd.Reqs = append(cmd.Reqs, req)
-		m.driver.requestsToSend = append(m.driver.requestsToSend, req)
+		m.awaitingReqs = append(m.awaitingReqs, req)
+		// m.driver.requestsToSend = append(m.driver.requestsToSend, req)
 
 		sizeLeft -= sizeToCopy
 		addr += sizeToCopy
@@ -123,6 +133,8 @@ func (m *defaultMemoryCopyMiddleware) processMemCopyD2HCommand(
 		m.driver.logTaskToGPUInitiate(now, cmd, req)
 	}
 
+	m.cyclesLeft = m.cyclesPerD2H
+
 	queue.IsRunning = true
 	return true
 }
@@ -177,17 +189,29 @@ func (m *defaultMemoryCopyMiddleware) sendFlushRequest(
 func (m *defaultMemoryCopyMiddleware) Tick(
 	now sim.VTimeInSec,
 ) (madeProgress bool) {
+	madeProgress = false
+
+	if m.cyclesLeft > 0 {
+		m.cyclesLeft--
+		madeProgress = true
+	} else if m.cyclesLeft == 0 {
+		m.driver.requestsToSend = append(m.driver.requestsToSend, m.awaitingReqs...)
+		m.awaitingReqs = nil
+		m.cyclesLeft = -1
+		madeProgress = true
+	}
+
 	req := m.driver.gpuPort.Peek()
 	if req == nil {
-		return false
+		return madeProgress
 	}
 
 	switch req := req.(type) {
 	case *sim.GeneralRsp:
-		return m.processGeneralRsp(now, req)
+		madeProgress = m.processGeneralRsp(now, req)
 	}
 
-	return false
+	return madeProgress
 }
 
 func (m *defaultMemoryCopyMiddleware) processGeneralRsp(

diff --git a/samples/runner/timingplatform.go b/samples/runner/timingplatform.go
@@ -183,6 +183,8 @@ func (b R9NanoPlatformBuilder) buildGPUDriver(
 		WithPageTable(pageTable).
 		WithLog2PageSize(b.log2PageSize).
 		WithGlobalStorage(b.globalStorage).
+		WithD2HCycles(8500).
+		WithH2DCycles(14500).
 		Build("Driver")
 	if b.visTracer != nil {
 		tracing.CollectTrace(gpuDriver, b.visTracer)
@@ -292,7 +294,7 @@ func (b R9NanoPlatformBuilder) createConnection(
 	// connection.SrcBufferCapacity = 40960000
 	pcieConnector := pcie.NewConnector().
 		WithEngine(engine).
-		WithVersion(3, 16).
+		WithVersion(4, 16).
 		WithSwitchLatency(140)
 
 	if b.visTracer != nil {