go

Migrations for Golang

Tagged golang, go, migrations  Languages go
package main

import (
    "database/sql"
    "fmt"
    _ "github.com/lib/pq" // Postgres
    "io/ioutil"
    "os"
    "regexp"
    "sort"
    "strconv"
    "strings"
)

type Migration struct {
    Name    string
    File    string
    Version int
}

type Migrations []Migration

func (slice Migrations) Len() int {
    return len(slice)
}

func (slice Migrations) Less(i, j int) bool {
    return slice[i].Version < slice[j].Version
}

func (slice Migrations) Swap(i, j int) {
    slice[i], slice[j] = slice[j], slice[i]
}

func migrate() {
    tx, err := db.Begin()
    dirname := "db/migrate/"
    // Find migration files
    d, err := os.Open(dirname)
    check(err)
    defer d.Close()
    fi, err := d.Readdir(-1)
    check(err)
    migrations := Migrations{}
    for _, file := range fi {
        if file.Mode().IsRegular() && strings.HasSuffix(file.Name(), ".sql") {
            // Find version number from file name 01_name.sql
            re := regexp.MustCompile("^[0-9]+")
            version, err := strconv.Atoi(re.FindString(file.Name()))
            if err != nil {
                panic("not a valid migration filename: " + file.Name())
            }
            migrations = append(migrations, Migration{Name: file.Name(), Version: version, File: file.Name()})
        }
    }
    // Process all migrations
    sort.Sort(migrations)
    for _, migration := range migrations {
        // Check if migration has been run
        var exists bool
        err = tx.QueryRow("select exists (select 1 from versions where id = $1)", migration.Version).Scan(&exists)
        if err != nil && err != sql.ErrNoRows {
            check(err)
        }
        // Run migration
        if exists == false {
            bytes, err := ioutil.ReadFile(dirname + migration.File)
            check(err)
            sql := string(bytes)
            // Split migration file at ; because some drivers can't handle multiple
            // statements This is not fool proof, but should work for most
            // migrations.
            count := strings.Count(sql, ";")
            statements := strings.SplitN(string(sql), ";", count)
            fmt.Println("RUN", "-", migration.Name)
            for _, statement := range statements {
                _, err = tx.Exec(statement)
                check(err)
            }
            _, err = tx.Exec("INSERT INTO versions VALUES ($1)", migration.Version)
            check(err)
        } else {
            fmt.Println("OK ", "-", migration.Name)
        }
    }
    check(tx.Commit())
}

Alternatives

There are several alternatives written in go. However, they all seem like hacks because of a limitation in golang’s database library/driver that makes executing raw SQL difficult.

Therefore I would recommend using sqitch, which is a standalone migration tool:

Sanitizing XML with Go's UnmarshalXML

Tagged unmarshal, go, xml  Languages go

Use a custom type to avoid this error when unmarshalling XML with Go:

strconv.ParseInt: parsing "86148865.00": invalid syntax

Example:

import (
    "encoding/xml"
    "strconv"
)

type DogHouse struct {
    //Count int xml:"dog_house>count" << this code will fail with "invalid syntax"
    Count sanitizedInt xml:"dog_house>count"
}

type sanitizedInt int

func (si *sanitizedInt) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
    var value string
    // Read tag content into value
    d.DecodeElement(&value, &start)
    // Remove "crap" and convert to int64
    i, err := strconv.ParseInt(strings.Replace(value, "crap", "", -1), 0, 64)
    if err != nil {
        return err
    }
    // Cast int64 to sanitizedInt
    *si = (sanitizedInt)(i)
    return nil
}

How to parse an XML document in Go

Tagged encoding, golang, rss, parser, go, xml  Languages go

This example shows how to fetch and parse an XML feed with Go.

Save this in main_test.go:

package main

import (
    "bytes"
    "code.google.com/p/go-charset/charset"
    _ "code.google.com/p/go-charset/data" // Import charset configuration files
    "encoding/xml"
    "io/ioutil"
    "log"
    "net/http"
    "testing"
)

type RssFeed struct {
    XMLName xml.Name  `xml:"rss"`
    Items   []RssItem `xml:"channel>item"`
}

type RssItem struct {
    XMLName     xml.Name `xml:"item"`
    Title       string   `xml:"title"`
    Link        string   `xml:"link"`
    Description string   `xml:"description"`
    //NestedTag    string      xml:">nested>tags>"
}

func fetchURL(url string) []byte {
    resp, err := http.Get(url)
    if err != nil {
        log.Fatalf("unable to GET '%s': %s", url, err)
    }
    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        log.Fatalf("unable to read body '%s': %s", url, err)
    }
    return body
}

func parseXML(xmlDoc []byte, target interface{}) {
    reader := bytes.NewReader(xmlDoc)
    decoder := xml.NewDecoder(reader)
    // Fixes "xml: encoding \"windows-1252\" declared but Decoder.CharsetReader is nil"
    decoder.CharsetReader = charset.NewReader
    if err := decoder.Decode(target); err != nil {
        log.Fatalf("unable to parse XML '%s':\n%s", err, xmlDoc)
    }
}

func TestParseReport(t *testing.T) {
    var rssFeed = &RssFeed{}
    xmlDoc := fetchURL("https://news.ycombinator.com/rss")
    parseXML(xmlDoc, &rssFeed)
    for _, item := range rssFeed.Items {
        log.Printf("%s: %s", item.Title, item.Link)
    }
}

Run the code with go test.

Golang Worker Pattern Example

Tagged go, pattern, worker, channel, golang  Languages go

An example of the worker pattern in Go.

Executes, e.g., 20 units of work with 20 workers, taking 2 seconds each, in 2 seconds.

package main

//
// $ go build worker.go && WORKERS=20 JOBS=20 ./worker
//
import (
 "fmt"
 "os"
 "strconv"
 "sync"
 "time"
)

// A unit of work
type Rabbit struct {
 ID   int
 Name string
}

// The worker accepting a unit of work
type RabbitWork struct {
 Rabbit Rabbit
}

func main() {
 jobCount, _ :=     strconv.Atoi(os.Getenv("JOBS"))
 workerCount, _ :=  strconv.Atoi(os.Getenv("WORKERS"))
 start := time.Now()
 workers := make(chan RabbitWork, workers)
 var wg sync.WaitGroup

 // Add  workers to our startup "Icarus"
 for i := 0; i < workerCount; i++ {
  wg.Add(1)
  go func() {
   for work := range workers {
    // do some work on data
    var rabbit = work.Rabbit
    fmt.Println("Capturing", rabbit.Name, "(", rabbit.ID, ")")
    time.Sleep(2 * time.Second)
    fmt.Println("Skinned", rabbit.Name, "(", rabbit.ID, ")")
   }
   wg.Done()
  }()
 }

 // Give some work to the workers
 for i := 1; i <= jobCount; i++ {
  workers <- RabbitWork{
   Rabbit: Rabbit{Name: "Toffe", ID: i},
  }
 }

 // Wait for workers to do their job
 close(workers)
 wg.Wait()
 fmt.Println("Work took", time.Since(start).Seconds(), "seconds")
}

Notes

  1. WaitGroup waits for all work to finish
  2. wg.Wait() blocks until wg.Done() is called 20 times
  3. workers <- RabbitWork will block if all 20 workers are busy working
  4. for data := range tasks iterates over incoming work sent through the tasks channel
  5. close(workers) closes the channel
  6. 40 units of work with 20 workers will complete in 4 seconds
  7. also see: http://marcio.io/2015/07/handling-1-million-requests-per-minute-with-golang/

Checking if a row exists in Go (database/sql and SQLX)

Tagged go, golang, sqlx  Languages go

Code:

func rowExists(query string, args ...interface{}) bool {
    var exists bool
    query = fmt.Sprintf("SELECT exists (%s)", query)
    err := db.QueryRow(query, args...).Scan(&exists)
    if err != nil && err != sql.ErrNoRows {
            glog.Fatalf("error checking if row exists '%s' %v", args, err)
    }
    return exists
}

Usage:

if rowExists("SELECT id FROM feed_items WHERE url=$1", item.Link) {
    return
}

How to Join Two Tables with jmoiron/sqlx

Tagged join, golang, go, sqlx  Languages go

Here’s an example of how to define relationships between structs when using SQL joins and sqlx:

type Feed struct {
  ID        int       `json:"id"`
  Title     string    `json:"title"`
  URL       string    `json:"url"`
  CreatedAt time.Time `json:"created_at" db:"created_at"`
}

type FeedItem struct {
  ID          int            `json:"id"`
  Title       string         `json:"title"`
  Description sql.NullString `json:"description"`
  URL         string         `json:"url"`
  CreatedAt   time.Time      `json:"created_at" db:"created_at"`
  FeedId      int            `json:"feed_id" db:"feed_id"`
  Feed        `db:"feed"` // << NOTE PREFIX 
}

func getFeeds() (*[]Feed, error) {
  feeds := []Feed{}
  err := db.Select(&feeds, "SELECT * FROM feeds")
  return &feeds, err
}

func getFeedItems() (*[]FeedItem, error) {
  items := []FeedItem{}
  sql := `SELECT
      feed_items.*,
      feeds.id "feed.id",
      feeds.title "feed.title",
      feeds.url "feed.url",
      feeds.created_at "feed.created_at"
    FROM
      feed_items JOIN feeds ON feed_items.feed_id = feeds.id;`
  err := db.Select(&items, sql)
  return &items, err
}

Go and Postgres Example

Tagged example, golang, polling, postgres  Languages go, bash
package main

import (
    "database/sql"
    _ "github.com/lib/pq"
    "log"
    "time"
)

func pollChanges() {
    var id int
    var name string
    rows, err := db.Query("SELECT id, name FROM people")
    if err != nil {
        log.Fatal(err)
    }
    defer rows.Close()
    for rows.Next() {
        err := rows.Scan(&id, &name)
        if err != nil {
            log.Fatal(err)
        }
        log.Println(id, name)
    }
    err = rows.Err()
    if err != nil {
        log.Fatal(err)
    }
}

var db *sql.DB

func main() {
    var err error
    db, err = sql.Open("postgres", "")
    if err != nil {
        log.Fatal(err)
    }
    defer db.Close()
    for {
        time.Sleep(1000 * time.Millisecond)
        log.Println("Polling for changes...")
        pollChanges()
    }
}

Run:

PGUSER= PGPASSWORD= PGDATABASE=db_xyz PGSSLMODE=disable ./go-app

Notes

Using question marks instead of, e.g., $1 will result in the following error:

syntax error at or near ","
  • Scan requires a pointer. Forgetting to add an ampersand (Scan(id) vs Scan(&id)) will result in the following error:
Scan error on column index 0: destination not a pointer

A simple log wrapper for Go

Tagged go, log, logging  Languages go

Version 1

This is a simple log wrapper for go’s built in log package. Put the following code in a file named log.go:

package main

import (
    golog "log"
    "os"
)

//
// Use ioutil.Discard instead of f to send log to '/dev/null'
//
type Logger struct {
    Info     *golog.Logger
    Error    *golog.Logger
    Debug    *golog.Logger
    FatalLog *golog.Logger
}
func (logger Logger) Fatal(format string, args ...interface{}) {
    logger.FatalLog.Printf(format, args...)
    golog.Fatalf(format, args...)
}

var logFile *os.File
var log Logger

func init() {
    fileName := os.Getenv("LOG")
    if len(fileName) > 0 {
        var err error
        logFile, err = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
        if err != nil {
            golog.Fatalf("error opening file: %v", err)
        }
    } else {
        logFile = os.Stdout
    }
    log.Info = golog.New(logFile, "INFO  ", golog.LstdFlags)
    log.Error = golog.New(logFile, "ERROR ", golog.LstdFlags)
    log.Debug = golog.New(logFile, "DEBUG ", golog.LstdFlags)
    log.FatalLog = golog.New(logFile, "FATAL ", golog.LstdFlags)
}

Example:

log.Info.Printf("Notifications %v", notifications)
log.Debug.Printf("Notifications %v", notifications)

Version 2

To remove the call to Printf you could do this:

type Log struct {
...
    //Info     *golog.Logger
    Info     func(string, ...interface{})
...
}

func init() {
...
    log.Info = golog.New(logFile, "INFO  ", golog.LstdFlags).Printf
...
}

Now you can simply call log.Info:

log.Info("Notifications %v", notifications)
log.Debug("Notifications %v", notifications)

See https://golang.org/pkg/log/#Logger

XPath with namespaces example in Go

Tagged go, xml, xpath  Languages go
package main

import (
    "fmt"
    "github.com/moovweb/gokogiri"
    "github.com/moovweb/gokogiri/xpath"
    "io/ioutil"
)

func main() {
    xml, _ := ioutil.ReadFile("elrc-20130531.xml")
    doc, _ := gokogiri.ParseXml(xml)
    defer doc.Free()
    xp := doc.DocXPathCtx()
    xp.RegisterNamespace("xbrli", "http://www.xbrl.org/2003/instance")
    x := xpath.Compile("//xbrli:context")
    groups, err := doc.Search(x)
    if err != nil {
        fmt.Println(err)
    }
    for i, group := range groups {
        fmt.Println(i, group.Content())
    }
}
go get
go run main.go

Simple content negotation in Golang

Tagged accept, go  Languages go
func DoSomething(r *http.Request) string {
    accept := r.Header.Get("Accept")
    switch accept {
    case "application/json":
        return renderJSON()
    case "application/xml":
        return renderXML()
    }
    return renderHTML()
}