go snippets

Sanitizing XML with Go's UnmarshalXML

Tagged unmarshal, xml, go  Languages go

Use a custom type to avoid this error when unmarshalling XML with Go:

strconv.ParseInt: parsing "86148865.00": invalid syntax

Example:

import (
    "encoding/xml"
    "strconv"
)

type DogHouse struct {
    //Count int xml:"dog_house>count" << this code will fail with "invalid syntax"
    Count sanitizedInt xml:"dog_house>count"
}

type sanitizedInt int

func (si *sanitizedInt) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
    var value string
    // Read tag content into value
    d.DecodeElement(&value, &start)
    // Remove "crap" and convert to int64
    i, err := strconv.ParseInt(strings.Replace(value, "crap", "", -1), 0, 64)
    if err != nil {
        return err
    }
    // Cast int64 to sanitizedInt
    *si = (sanitizedInt)(i)
    return nil
}

Golang Worker Pattern Example

Tagged go, pattern, worker, channel, golang  Languages go

An example of the worker pattern in Go.

Executes, e.g., 20 units of work with 20 workers, taking 2 seconds each, in 2 seconds.

package main

//
// $ go build worker.go && WORKERS=20 JOBS=20 ./worker
//
import (
 "fmt"
 "os"
 "strconv"
 "sync"
 "time"
)

// A unit of work
type Rabbit struct {
 ID   int
 Name string
}

// The worker accepting a unit of work
type RabbitWork struct {
 Rabbit Rabbit
}

func main() {
 jobCount, _ :=     strconv.Atoi(os.Getenv("JOBS"))
 workerCount, _ :=  strconv.Atoi(os.Getenv("WORKERS"))
 start := time.Now()
 workers := make(chan RabbitWork, workers)
 var wg sync.WaitGroup

 // Add  workers to our startup "Icarus"
 for i := 0; i < workerCount; i++ {
  wg.Add(1)
  go func() {
   for work := range workers {
    // do some work on data
    var rabbit = work.Rabbit
    fmt.Println("Capturing", rabbit.Name, "(", rabbit.ID, ")")
    time.Sleep(2 * time.Second)
    fmt.Println("Skinned", rabbit.Name, "(", rabbit.ID, ")")
   }
   wg.Done()
  }()
 }

 // Give some work to the workers
 for i := 1; i <= jobCount; i++ {
  workers <- RabbitWork{
   Rabbit: Rabbit{Name: "Toffe", ID: i},
  }
 }

 // Wait for workers to do their job
 close(workers)
 wg.Wait()
 fmt.Println("Work took", time.Since(start).Seconds(), "seconds")
}

Notes

  1. WaitGroup waits for all work to finish
  2. wg.Wait() blocks until wg.Done() is called 20 times
  3. workers <- RabbitWork will block if all 20 workers are busy working
  4. for data := range tasks iterates over incoming work sent through the tasks channel
  5. close(workers) closes the channel
  6. 40 units of work with 20 workers will complete in 4 seconds
  7. also see: http://marcio.io/2015/07/handling-1-million-requests-per-minute-with-golang/

How to Join Two Tables with jmoiron/sqlx

Tagged join, golang, go, sqlx  Languages go

Here’s an example of how to define relationships between structs when using SQL joins and sqlx:

type Feed struct {
  ID        int       `json:"id"`
  Title     string    `json:"title"`
  URL       string    `json:"url"`
  CreatedAt time.Time `json:"created_at" db:"created_at"`
}

type FeedItem struct {
  ID          int            `json:"id"`
  Title       string         `json:"title"`
  Description sql.NullString `json:"description"`
  URL         string         `json:"url"`
  CreatedAt   time.Time      `json:"created_at" db:"created_at"`
  FeedId      int            `json:"feed_id" db:"feed_id"`
  Feed        `db:"feed"` // << NOTE PREFIX 
}

func getFeeds() (*[]Feed, error) {
  feeds := []Feed{}
  err := db.Select(&feeds, "SELECT * FROM feeds")
  return &feeds, err
}

func getFeedItems() (*[]FeedItem, error) {
  items := []FeedItem{}
  sql := `SELECT
      feed_items.*,
      feeds.id "feed.id",
      feeds.title "feed.title",
      feeds.url "feed.url",
      feeds.created_at "feed.created_at"
    FROM
      feed_items JOIN feeds ON feed_items.feed_id = feeds.id;`
  err := db.Select(&items, sql)
  return &items, err
}

How to parse an XML document in Go

Tagged encoding, golang, rss, parser, go, xml  Languages go

This example shows how to fetch and parse an XML feed with Go.

Save this in main_test.go:

package main

import (
    "bytes"
    "code.google.com/p/go-charset/charset"
    _ "code.google.com/p/go-charset/data" // Import charset configuration files
    "encoding/xml"
    "io/ioutil"
    "log"
    "net/http"
    "testing"
)

type RssFeed struct {
    XMLName xml.Name  `xml:"rss"`
    Items   []RssItem `xml:"channel>item"`
}

type RssItem struct {
    XMLName     xml.Name `xml:"item"`
    Title       string   `xml:"title"`
    Link        string   `xml:"link"`
    Description string   `xml:"description"`
    //NestedTag    string      xml:">nested>tags>"
}

func fetchURL(url string) []byte {
    resp, err := http.Get(url)
    if err != nil {
        log.Fatalf("unable to GET '%s': %s", url, err)
    }
    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        log.Fatalf("unable to read body '%s': %s", url, err)
    }
    return body
}

func parseXML(xmlDoc []byte, target interface{}) {
    reader := bytes.NewReader(xmlDoc)
    decoder := xml.NewDecoder(reader)
    // Fixes "xml: encoding \"windows-1252\" declared but Decoder.CharsetReader is nil"
    decoder.CharsetReader = charset.NewReader
    if err := decoder.Decode(target); err != nil {
        log.Fatalf("unable to parse XML '%s':\n%s", err, xmlDoc)
    }
}

func TestParseReport(t *testing.T) {
    var rssFeed = &RssFeed{}
    xmlDoc := fetchURL("https://news.ycombinator.com/rss")
    parseXML(xmlDoc, &rssFeed)
    for _, item := range rssFeed.Items {
        log.Printf("%s: %s", item.Title, item.Link)
    }
}

Run the code with go test.

Migrations for Golang

Tagged go, golang, migrations  Languages go
package main

import (
    "database/sql"
    "fmt"
    _ "github.com/lib/pq" // Postgres
    "io/ioutil"
    "os"
    "regexp"
    "sort"
    "strconv"
    "strings"
)

type Migration struct {
    Name    string
    File    string
    Version int
}

type Migrations []Migration

func (slice Migrations) Len() int {
    return len(slice)
}

func (slice Migrations) Less(i, j int) bool {
    return slice[i].Version < slice[j].Version
}

func (slice Migrations) Swap(i, j int) {
    slice[i], slice[j] = slice[j], slice[i]
}

func migrate() {
    tx, err := db.Begin()
    dirname := "db/migrate/"
    // Find migration files
    d, err := os.Open(dirname)
    check(err)
    defer d.Close()
    fi, err := d.Readdir(-1)
    check(err)
    migrations := Migrations{}
    for _, file := range fi {
        if file.Mode().IsRegular() && strings.HasSuffix(file.Name(), ".sql") {
            // Find version number from file name 01_name.sql
            re := regexp.MustCompile("^[0-9]+")
            version, err := strconv.Atoi(re.FindString(file.Name()))
            if err != nil {
                panic("not a valid migration filename: " + file.Name())
            }
            migrations = append(migrations, Migration{Name: file.Name(), Version: version, File: file.Name()})
        }
    }
    // Process all migrations
    sort.Sort(migrations)
    for _, migration := range migrations {
        // Check if migration has been run
        var exists bool
        err = tx.QueryRow("select exists (select 1 from versions where id = $1)", migration.Version).Scan(&exists)
        if err != nil && err != sql.ErrNoRows {
            check(err)
        }
        // Run migration
        if exists == false {
            bytes, err := ioutil.ReadFile(dirname + migration.File)
            check(err)
            sql := string(bytes)
            // Split migration file at ; because some drivers can't handle multiple
            // statements This is not fool proof, but should work for most
            // migrations.
            count := strings.Count(sql, ";")
            statements := strings.SplitN(string(sql), ";", count)
            fmt.Println("RUN", "-", migration.Name)
            for _, statement := range statements {
                _, err = tx.Exec(statement)
                check(err)
            }
            _, err = tx.Exec("INSERT INTO versions VALUES ($1)", migration.Version)
            check(err)
        } else {
            fmt.Println("OK ", "-", migration.Name)
        }
    }
    check(tx.Commit())
}

Alternatives

There are several alternatives written in go. However, they all seem like hacks because of a limitation in golang’s database library/driver that makes executing raw SQL difficult.

Therefore I would recommend using sqitch, which is a standalone migration tool:

Checking if a row exists in Go (database/sql and SQLX)

Tagged go, golang, sqlx  Languages go

Code:

func rowExists(query string, args ...interface{}) bool {
    var exists bool
    query = fmt.Sprintf("SELECT exists (%s)", query)
    err := db.QueryRow(query, args...).Scan(&exists)
    if err != nil && err != sql.ErrNoRows {
            glog.Fatalf("error checking if row exists '%s' %v", args, err)
    }
    return exists
}

Usage:

if rowExists("SELECT id FROM feed_items WHERE url=$1", item.Link) {
    return
}

A simple log wrapper for Go

Tagged go, log, logging  Languages go

Version 1

This is a simple log wrapper for go’s built in log package. Put the following code in a file named log.go:

package main

import (
    golog "log"
    "os"
)

//
// Use ioutil.Discard instead of f to send log to '/dev/null'
//
type Logger struct {
    Info     *golog.Logger
    Error    *golog.Logger
    Debug    *golog.Logger
    FatalLog *golog.Logger
}
func (logger Logger) Fatal(format string, args ...interface{}) {
    logger.FatalLog.Printf(format, args...)
    golog.Fatalf(format, args...)
}

var logFile *os.File
var log Logger

func init() {
    fileName := os.Getenv("LOG")
    if len(fileName) > 0 {
        var err error
        logFile, err = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
        if err != nil {
            golog.Fatalf("error opening file: %v", err)
        }
    } else {
        logFile = os.Stdout
    }
    log.Info = golog.New(logFile, "INFO  ", golog.LstdFlags)
    log.Error = golog.New(logFile, "ERROR ", golog.LstdFlags)
    log.Debug = golog.New(logFile, "DEBUG ", golog.LstdFlags)
    log.FatalLog = golog.New(logFile, "FATAL ", golog.LstdFlags)
}

Example:

log.Info.Printf("Notifications %v", notifications)
log.Debug.Printf("Notifications %v", notifications)

Version 2

To remove the call to Printf you could do this:

type Log struct {
...
    //Info     *golog.Logger
    Info     func(string, ...interface{})
...
}

func init() {
...
    log.Info = golog.New(logFile, "INFO  ", golog.LstdFlags).Printf
...
}

Now you can simply call log.Info:

log.Info("Notifications %v", notifications)
log.Debug("Notifications %v", notifications)

See https://golang.org/pkg/log/#Logger

How to deploy a Go app through SSH

Tagged go, golang, ssh  Languages bash
# Target = Linux
GOARCH=amd64 GOOS=linux go build -o dist/go-fcuk-linux
scp dist/go-fcuk 127.0.0.1:/tmp
ssh -t 127.0.0.1 "sudo mv /tmp/go-fcuk-linux /usr/local/bin/go-fcuk"

XPath with namespaces example in Go

Tagged go, xml, xpath  Languages go
package main

import (
    "fmt"
    "github.com/moovweb/gokogiri"
    "github.com/moovweb/gokogiri/xpath"
    "io/ioutil"
)

func main() {
    xml, _ := ioutil.ReadFile("elrc-20130531.xml")
    doc, _ := gokogiri.ParseXml(xml)
    defer doc.Free()
    xp := doc.DocXPathCtx()
    xp.RegisterNamespace("xbrli", "http://www.xbrl.org/2003/instance")
    x := xpath.Compile("//xbrli:context")
    groups, err := doc.Search(x)
    if err != nil {
        fmt.Println(err)
    }
    for i, group := range groups {
        fmt.Println(i, group.Content())
    }
}
go get
go run main.go

Simple content negotation in Golang

Tagged accept, go  Languages go
func DoSomething(r *http.Request) string {
    accept := r.Header.Get("Accept")
    switch accept {
    case "application/json":
        return renderJSON()
    case "application/xml":
        return renderXML()
    }
    return renderHTML()
}