Golang Worker Pattern Example

Tagged golang, go, worker, pattern, channel  Languages go

An example of the worker pattern in Go.

Executes, e.g., 20 units of work with 20 workers, taking 2 seconds each, in 2 seconds.

package main

//
// $ go build worker.go && WORKERS=20 JOBS=20 ./worker
//
import (
 "fmt"
 "os"
 "strconv"
 "sync"
 "time"
)

// A unit of work
type Rabbit struct {
 ID   int
 Name string
}

// The worker accepting a unit of work
type RabbitWork struct {
 Rabbit Rabbit
}

func main() {
 jobCount, _ :=     strconv.Atoi(os.Getenv("JOBS"))
 workerCount, _ :=  strconv.Atoi(os.Getenv("WORKERS"))
 start := time.Now()
 workers := make(chan RabbitWork, workers)
 var wg sync.WaitGroup

 // Add  workers to our startup "Icarus"
 for i := 0; i < workerCount; i++ {
  wg.Add(1)
  go func() {
   for work := range workers {
    // do some work on data
    var rabbit = work.Rabbit
    fmt.Println("Capturing", rabbit.Name, "(", rabbit.ID, ")")
    time.Sleep(2 * time.Second)
    fmt.Println("Skinned", rabbit.Name, "(", rabbit.ID, ")")
   }
   wg.Done()
  }()
 }

 // Give some work to the workers
 for i := 1; i <= jobCount; i++ {
  workers <- RabbitWork{
   Rabbit: Rabbit{Name: "Toffe", ID: i},
  }
 }

 // Wait for workers to do their job
 close(workers)
 wg.Wait()
 fmt.Println("Work took", time.Since(start).Seconds(), "seconds")
}

Notes

  1. WaitGroup waits for all work to finish
  2. wg.Wait() blocks until wg.Done() is called 20 times
  3. workers <- RabbitWork will block if all 20 workers are busy working
  4. for data := range tasks iterates over incoming work sent through the tasks channel
  5. close(workers) closes the channel
  6. 40 units of work with 20 workers will complete in 4 seconds
  7. also see: http://marcio.io/2015/07/handling-1-million-requests-per-minute-with-golang/