Go Scheduler Internals: G-M-P Deep Dive

1,000,000 goroutines
        ↓
      ???
        ↓
    8 CPU cores

┌─────────────────────────────────────────────────────────┐
│                      Go Runtime                         │
│                                                         │
│  ┌─────────┐  ┌─────────┐  ┌─────────┐  ┌─────────┐  │
│  │    G    │  │    G    │  │    G    │  │    G    │  │  G = Goroutine
│  └────┬────┘  └────┬────┘  └────┬────┘  └────┬────┘  │  (task to execute)
│       │            │            │            │        │
│  ┌────▼────────────▼────────────▼────────────▼─────┐ │
│  │                     P                            │ │  P = Processor
│  │  (Local run queue + execution context)          │ │  (token to run)
│  └────┬─────────────────────────────────────────────┘ │
│       │                                                │
│  ┌────▼────┐                                          │
│  │    M    │                                          │  M = Machine
│  └────┬────┘                                          │  (OS thread)
│       │                                                │
└───────┼────────────────────────────────────────────────┘
        │
   ┌────▼────┐
   │   CPU   │
   └─────────┘

type g struct {
    stack       stack       // Stack memory
    stackguard0 uintptr     // Stack overflow detection
    m           *m          // Current M running this G
    sched       gobuf       // Saved registers (PC, SP)
    atomicstatus uint32     // State: runnable, running, waiting, dead
    goid        int64       // Goroutine ID
    waitsince   int64       // Time spent waiting
    lockedm     *m          // Locked to specific M?
}

type m struct {
    g0      *g          // Goroutine for scheduling (not user code)
    curg    *g          // Current user goroutine
    p       *p          // Current P (can be nil)
    nextp   *p          // Next P to run after syscall
    id      int64
    spinning bool       // Looking for work?
    park    note        // Sleep/wake mechanism
    alllink *m          // Link in allm list
}

type p struct {
    id          int32
    status      uint32      // _Pidle, _Prunning, _Psyscall
    link        *p
    m           *m          // Current M owning this P
    runqhead    uint32      // Local run queue head
    runqtail    uint32      // Local run queue tail
    runq        [256]*g     // Local run queue (circular buffer)
    runnext     *g          // Next G to run (priority)
    
    // For GC
    mcache      *mcache
    
    // Stats
    schedtick   uint32      // Number of schedules
}

# Check GOMAXPROCS
go env GOMAXPROCS

# Set in code
runtime.GOMAXPROCS(8)

# Set via env
GOMAXPROCS=8 go run main.go

go func() {
    fmt.Println("Hello")
}()

M với P đang idle, tìm G để chạy:

1. Check P.runnext (priority slot)
   ↓ None
2. Check local run queue (P.runq)
   ↓ Empty
3. Check global run queue
   ↓ Empty
4. Steal from other P (work stealing)
   ↓ All empty
5. Check network poller (netpoll)
   ↓ None
6. M goes to sleep

P1: [G1, G2, G3, G4, G5, G6]
P2: []

→ Work stealing

P1: [G1, G2, G3]
P2: [G4, G5, G6]

M + P + G
  ↓
Execute G's code
  ↓
One of:
  - G finishes → pick next G
  - G blocks (channel, syscall) → park G, pick next G
  - G yields (runtime.Gosched) → put back to queue, pick next G
  - Preemption signal → save state, pick next G

M1 + P1 + G1 (calling syscall)
  ↓
G1 enters syscall
  ↓
P1 detaches from M1
  ↓
P1 finds M2 (or creates new M)
  ↓
M2 + P1 continues scheduling other Gs
  ↓
M1 waits for syscall to finish
  ↓
Syscall done → M1 tries to reacquire P
  - Success: M1 + P1 + G1 continues
  - Fail: G1 put in global queue, M1 sleeps

G1 waiting on channel
  ↓
G1 state = _Gwaiting
  ↓
G1 put in channel's wait queue
  ↓
M picks next G from run queue
  ↓
... later ...
  ↓
Another G sends to channel
  ↓
G1 woken up → state = _Grunnable
  ↓
G1 back to run queue

func foo() {
    // Compiler injects: check preemption signal
    bar()
}

// Goroutine này monopolize CPU
for {
    i++
}

// Go 1.14+: vẫn có thể preempt
for {
    i++
}

Sysmon loop (chạy mỗi 20µs → 10ms adaptive):
  ↓
1. Check retake Ps in syscall > 10ms
2. Check preempt Gs running > 10ms
3. Check force GC if idle > 2min
4. Poll network if no active M
  ↓
Sleep, then repeat

G1: conn.Read()
  ↓
Syscall would block
  ↓
Runtime puts G1 in netpoll wait list
  ↓
M picks next G (G2)
  ↓
... G2 runs ...
  ↓
Netpoller detects conn is ready
  ↓
G1 moved to run queue
  ↓
M eventually picks G1 again
  ↓
G1 resumes, Read() returns

func cpuBound() {
    for i := 0; i < 1e9; i++ {
        _ = i * i
    }
}

func main() {
    runtime.GOMAXPROCS(runtime.NumCPU())
    
    for i := 0; i < 100; i++ {
        go cpuBound()
    }
    
    time.Sleep(10 * time.Second)
}

GODEBUG=schedtrace=1000 ./myapp

SCHED 0ms: gomaxprocs=8 idleprocs=0 threads=10 spinningthreads=0 idlethreads=4 runqueue=0 [0 0 0 0 0 0 0 0]

fmt.Println("Goroutines:", runtime.NumGoroutine())

go tool pprof http://localhost:6060/debug/pprof/goroutine

(pprof) top
(pprof) list <function>
(pprof) traces

curl http://localhost:6060/debug/pprof/trace?seconds=5 > trace.out
go tool trace trace.out

func init() {
    runtime.LockOSThread()
}

func main() {
    // This goroutine always runs on same OS thread
    // Useful for C libraries requiring thread-local state
}

// ❌ BAD: Tạo 1 triệu goroutines cùng lúc
for i := 0; i < 1_000_000; i++ {
    go process(i)
}

// ✅ GOOD: Worker pool với giới hạn
const workers = 100
sem := make(chan struct{}, workers)

for i := 0; i < 1_000_000; i++ {
    sem <- struct{}{}
    go func(id int) {
        defer func() { <-sem }()
        process(id)
    }(i)
}

// ❌ BAD: Busy wait
for !ready {
    // Monopolizes CPU
}

// ✅ GOOD: Yield hoặc dùng channel
for !ready {
    runtime.Gosched()  // Yield to other goroutines
}

// ✅ BETTER: Dùng channel
<-readyChan

// ❌ BAD: Mỗi item một goroutine
for _, item := range items {
    go process(item)  // Scheduling overhead
}

// ✅ GOOD: Batch items
chunkSize := 100
for i := 0; i < len(items); i += chunkSize {
    end := i + chunkSize
    if end > len(items) {
        end = len(items)
    }
    
    chunk := items[i:end]
    go func(batch []Item) {
        for _, item := range batch {
            process(item)
        }
    }(chunk)
}