golang snippets

Web Microframework Benchmark

Tagged benchmark, elixir, clojure, golang, ruby  Languages 

Results on an old MacBook Air:

Ruby 2.1.0 + Hobbit + Hat + puma (-t 8 -w 2) - ~100-150 req/s (Hat = Hobbit app template with i18n, asset pipeline, etc)
Ruby 2.1.0 + Hobbit + Hat custom + puma (-t 8 -w 2) - ~1500 req/s (Hat without asset pipeline)
Ruby 2.1.0 + Hobbit + puma (-t 8 -w 2) - ~1600 req/s
Ruby 2.1.0 + rack + puma (-t 8 -w 2) - ~1600 req/s
Golang 1.3.1 + net/http - ~2700 req/s
Elixir 1.0.0 + Phoenix 0.4.1 - ~1300 req/s
Clojure 1.6.0 + ring 1.3.1 - ~5000 req/s
Clojure 1.6.0 + ring 1.3.1 + slim - ~270 req/s

YMMV.

Golang Worker Pattern Example

Tagged go, pattern, golang, worker, channel  Languages go

An example of the worker pattern in Go.

Executes, e.g., 20 units of work with 20 workers, taking 2 seconds each, in 2 seconds.

package main

//
// $ go build worker.go && WORKERS=20 JOBS=20 ./worker
//
import (
 "fmt"
 "os"
 "strconv"
 "sync"
 "time"
)

// A unit of work
type Rabbit struct {
 ID   int
 Name string
}

// The worker accepting a unit of work
type RabbitWork struct {
 Rabbit Rabbit
}

func main() {
 jobCount, _ :=     strconv.Atoi(os.Getenv("JOBS"))
 workerCount, _ :=  strconv.Atoi(os.Getenv("WORKERS"))
 start := time.Now()
 workers := make(chan RabbitWork, workers)
 var wg sync.WaitGroup

 // Add  workers to our startup "Icarus"
 for i := 0; i < workerCount; i++ {
  wg.Add(1)
  go func() {
   for work := range workers {
    // do some work on data
    var rabbit = work.Rabbit
    fmt.Println("Capturing", rabbit.Name, "(", rabbit.ID, ")")
    time.Sleep(2 * time.Second)
    fmt.Println("Skinned", rabbit.Name, "(", rabbit.ID, ")")
   }
   wg.Done()
  }()
 }

 // Give some work to the workers
 for i := 1; i <= jobCount; i++ {
  workers <- RabbitWork{
   Rabbit: Rabbit{Name: "Toffe", ID: i},
  }
 }

 // Wait for workers to do their job
 close(workers)
 wg.Wait()
 fmt.Println("Work took", time.Since(start).Seconds(), "seconds")
}

Notes

  1. WaitGroup waits for all work to finish
  2. wg.Wait() blocks until wg.Done() is called 20 times
  3. workers <- RabbitWork will block if all 20 workers are busy working
  4. for data := range tasks iterates over incoming work sent through the tasks channel
  5. close(workers) closes the channel
  6. 40 units of work with 20 workers will complete in 4 seconds
  7. also see: http://marcio.io/2015/07/handling-1-million-requests-per-minute-with-golang/

How to Join Two Tables with jmoiron/sqlx

Tagged go, golang, join, sqlx  Languages go

Here’s an example of how to define relationships between structs when using SQL joins and sqlx:

type Feed struct {
  ID        int       `json:"id"`
  Title     string    `json:"title"`
  URL       string    `json:"url"`
  CreatedAt time.Time `json:"created_at" db:"created_at"`
}

type FeedItem struct {
  ID          int            `json:"id"`
  Title       string         `json:"title"`
  Description sql.NullString `json:"description"`
  URL         string         `json:"url"`
  CreatedAt   time.Time      `json:"created_at" db:"created_at"`
  FeedId      int            `json:"feed_id" db:"feed_id"`
  Feed        `db:"feed"` // << NOTE PREFIX 
}

func getFeeds() (*[]Feed, error) {
  feeds := []Feed{}
  err := db.Select(&feeds, "SELECT * FROM feeds")
  return &feeds, err
}

func getFeedItems() (*[]FeedItem, error) {
  items := []FeedItem{}
  sql := `SELECT
      feed_items.*,
      feeds.id "feed.id",
      feeds.title "feed.title",
      feeds.url "feed.url",
      feeds.created_at "feed.created_at"
    FROM
      feed_items JOIN feeds ON feed_items.feed_id = feeds.id;`
  err := db.Select(&items, sql)
  return &items, err
}

How to parse an XML document in Go

Tagged encoding, go, xml, golang, rss, parser  Languages go

This example shows how to fetch and parse an XML feed with Go.

Save this in main_test.go:

package main

import (
    "bytes"
    "code.google.com/p/go-charset/charset"
    _ "code.google.com/p/go-charset/data" // Import charset configuration files
    "encoding/xml"
    "io/ioutil"
    "log"
    "net/http"
    "testing"
)

type RssFeed struct {
    XMLName xml.Name  `xml:"rss"`
    Items   []RssItem `xml:"channel>item"`
}

type RssItem struct {
    XMLName     xml.Name `xml:"item"`
    Title       string   `xml:"title"`
    Link        string   `xml:"link"`
    Description string   `xml:"description"`
    //NestedTag    string      xml:">nested>tags>"
}

func fetchURL(url string) []byte {
    resp, err := http.Get(url)
    if err != nil {
        log.Fatalf("unable to GET '%s': %s", url, err)
    }
    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        log.Fatalf("unable to read body '%s': %s", url, err)
    }
    return body
}

func parseXML(xmlDoc []byte, target interface{}) {
    reader := bytes.NewReader(xmlDoc)
    decoder := xml.NewDecoder(reader)
    // Fixes "xml: encoding \"windows-1252\" declared but Decoder.CharsetReader is nil"
    decoder.CharsetReader = charset.NewReader
    if err := decoder.Decode(target); err != nil {
        log.Fatalf("unable to parse XML '%s':\n%s", err, xmlDoc)
    }
}

func TestParseReport(t *testing.T) {
    var rssFeed = &RssFeed{}
    xmlDoc := fetchURL("https://news.ycombinator.com/rss")
    parseXML(xmlDoc, &rssFeed)
    for _, item := range rssFeed.Items {
        log.Printf("%s: %s", item.Title, item.Link)
    }
}

Run the code with go test.

Migrations for Golang

Tagged golang, go, migrations  Languages go
package main

import (
    "database/sql"
    "fmt"
    _ "github.com/lib/pq" // Postgres
    "io/ioutil"
    "os"
    "regexp"
    "sort"
    "strconv"
    "strings"
)

type Migration struct {
    Name    string
    File    string
    Version int
}

type Migrations []Migration

func (slice Migrations) Len() int {
    return len(slice)
}

func (slice Migrations) Less(i, j int) bool {
    return slice[i].Version < slice[j].Version
}

func (slice Migrations) Swap(i, j int) {
    slice[i], slice[j] = slice[j], slice[i]
}

func migrate() {
    tx, err := db.Begin()
    dirname := "db/migrate/"
    // Find migration files
    d, err := os.Open(dirname)
    check(err)
    defer d.Close()
    fi, err := d.Readdir(-1)
    check(err)
    migrations := Migrations{}
    for _, file := range fi {
        if file.Mode().IsRegular() && strings.HasSuffix(file.Name(), ".sql") {
            // Find version number from file name 01_name.sql
            re := regexp.MustCompile("^[0-9]+")
            version, err := strconv.Atoi(re.FindString(file.Name()))
            if err != nil {
                panic("not a valid migration filename: " + file.Name())
            }
            migrations = append(migrations, Migration{Name: file.Name(), Version: version, File: file.Name()})
        }
    }
    // Process all migrations
    sort.Sort(migrations)
    for _, migration := range migrations {
        // Check if migration has been run
        var exists bool
        err = tx.QueryRow("select exists (select 1 from versions where id = $1)", migration.Version).Scan(&exists)
        if err != nil && err != sql.ErrNoRows {
            check(err)
        }
        // Run migration
        if exists == false {
            bytes, err := ioutil.ReadFile(dirname + migration.File)
            check(err)
            sql := string(bytes)
            // Split migration file at ; because some drivers can't handle multiple
            // statements This is not fool proof, but should work for most
            // migrations.
            count := strings.Count(sql, ";")
            statements := strings.SplitN(string(sql), ";", count)
            fmt.Println("RUN", "-", migration.Name)
            for _, statement := range statements {
                _, err = tx.Exec(statement)
                check(err)
            }
            _, err = tx.Exec("INSERT INTO versions VALUES ($1)", migration.Version)
            check(err)
        } else {
            fmt.Println("OK ", "-", migration.Name)
        }
    }
    check(tx.Commit())
}

Alternatives

There are several alternatives written in go. However, they all seem like hacks because of a limitation in golang’s database library/driver that makes executing raw SQL difficult.

Therefore I would recommend using sqitch, which is a standalone migration tool:

Checking if a row exists in Go (database/sql and SQLX)

Tagged golang, go, sqlx  Languages go

Code:

func rowExists(query string, args ...interface{}) bool {
    var exists bool
    query = fmt.Sprintf("SELECT exists (%s)", query)
    err := db.QueryRow(query, args...).Scan(&exists)
    if err != nil && err != sql.ErrNoRows {
            glog.Fatalf("error checking if row exists '%s' %v", args, err)
    }
    return exists
}

Usage:

if rowExists("SELECT id FROM feed_items WHERE url=$1", item.Link) {
    return
}

Go and Postgres Example

Tagged postgres, polling, golang, example  Languages go, bash
package main

import (
    "database/sql"
    _ "github.com/lib/pq"
    "log"
    "time"
)

func pollChanges() {
    var id int
    var name string
    rows, err := db.Query("SELECT id, name FROM people")
    if err != nil {
        log.Fatal(err)
    }
    defer rows.Close()
    for rows.Next() {
        err := rows.Scan(&id, &name)
        if err != nil {
            log.Fatal(err)
        }
        log.Println(id, name)
    }
    err = rows.Err()
    if err != nil {
        log.Fatal(err)
    }
}

var db *sql.DB

func main() {
    var err error
    db, err = sql.Open("postgres", "")
    if err != nil {
        log.Fatal(err)
    }
    defer db.Close()
    for {
        time.Sleep(1000 * time.Millisecond)
        log.Println("Polling for changes...")
        pollChanges()
    }
}

Run:

PGUSER= PGPASSWORD= PGDATABASE=db_xyz PGSSLMODE=disable ./go-app

Notes

Using question marks instead of, e.g., $1 will result in the following error:

syntax error at or near ","
  • Scan requires a pointer. Forgetting to add an ampersand (Scan(id) vs Scan(&id)) will result in the following error:
Scan error on column index 0: destination not a pointer

How to deploy a Go app through SSH

Tagged go, golang, ssh  Languages bash
# Target = Linux
GOARCH=amd64 GOOS=linux go build -o dist/go-fcuk-linux
scp dist/go-fcuk 127.0.0.1:/tmp
ssh -t 127.0.0.1 "sudo mv /tmp/go-fcuk-linux /usr/local/bin/go-fcuk"