go

Migrations for Golang

Tagged sqitch, migrate, go, golang, migrations  Languages go

Alternative 1: Code your own

package main

import (
    "database/sql"
    "fmt"
    _ "github.com/lib/pq" // Postgres
    "io/ioutil"
    "os"
    "regexp"
    "sort"
    "strconv"
    "strings"
)

type Migration struct {
    Name    string
    File    string
    Version int
}

type Migrations []Migration

func (slice Migrations) Len() int {
    return len(slice)
}

func (slice Migrations) Less(i, j int) bool {
    return slice[i].Version < slice[j].Version
}

func (slice Migrations) Swap(i, j int) {
    slice[i], slice[j] = slice[j], slice[i]
}

func migrate() {
    tx, err := db.Begin()
    dirname := "db/migrate/"
    // Find migration files
    d, err := os.Open(dirname)
    check(err)
    defer d.Close()
    fi, err := d.Readdir(-1)
    check(err)
    migrations := Migrations{}
    for _, file := range fi {
        if file.Mode().IsRegular() && strings.HasSuffix(file.Name(), ".sql") {
            // Find version number from file name 01_name.sql
            re := regexp.MustCompile("^[0-9]+")
            version, err := strconv.Atoi(re.FindString(file.Name()))
            if err != nil {
                panic("not a valid migration filename: " + file.Name())
            }
            migrations = append(migrations, Migration{Name: file.Name(), Version: version, File: file.Name()})
        }
    }
    // Process all migrations
    sort.Sort(migrations)
    for _, migration := range migrations {
        // Check if migration has been run
        var exists bool
        err = tx.QueryRow("select exists (select 1 from versions where id = $1)", migration.Version).Scan(&exists)
        if err != nil && err != sql.ErrNoRows {
            check(err)
        }
        // Run migration
        if exists == false {
            bytes, err := ioutil.ReadFile(dirname + migration.File)
            check(err)
            sql := string(bytes)
            // Split migration file at ; because some drivers can't handle multiple
            // statements This is not fool proof, but should work for most
            // migrations.
            count := strings.Count(sql, ";")
            statements := strings.SplitN(string(sql), ";", count)
            fmt.Println("RUN", "-", migration.Name)
            for _, statement := range statements {
                _, err = tx.Exec(statement)
                check(err)
            }
            _, err = tx.Exec("INSERT INTO versions VALUES ($1)", migration.Version)
            check(err)
        } else {
            fmt.Println("OK ", "-", migration.Name)
        }
    }
    check(tx.Commit())
}

Alternative 2: sql/migrate

package main

import (
    "github.com/rubenv/sql-migrate"
    "gitlab.com/christianhellsten/a15/domains"
    "log"
    "os"
)

func main() {
    migrations := &migrate.FileMigrationSource{
        Dir: "db/migrations",
    }
    var dir migrate.MigrationDirection
    switch os.Args[1] {
    case "up":
        dir = migrate.Up
    case "down":
        dir = migrate.Down
    default:
        dir = migrate.Up
    }
    n, err := migrate.Exec(domains.DBI, "postgres", migrations, dir)
    if err != nil {
        log.Fatalln(err)
    }
    log.Printf("Applied %d migrations %s!\n", n, os.Args[1])
}

Alternative 3: Sqitch

http://sqitch.org/

Sanitizing XML with Go's UnmarshalXML

Tagged go, xml, unmarshal  Languages go

Use a custom type to avoid this error when unmarshalling XML with Go:

strconv.ParseInt: parsing "86148865.00": invalid syntax

Example:

import (
    "encoding/xml"
    "strconv"
)

type DogHouse struct {
    //Count int xml:"dog_house>count" << this code will fail with "invalid syntax"
    Count sanitizedInt xml:"dog_house>count"
}

type sanitizedInt int

func (si *sanitizedInt) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
    var value string
    // Read tag content into value
    d.DecodeElement(&value, &start)
    // Remove "crap" and convert to int64
    i, err := strconv.ParseInt(strings.Replace(value, "crap", "", -1), 0, 64)
    if err != nil {
        return err
    }
    // Cast int64 to sanitizedInt
    *si = (sanitizedInt)(i)
    return nil
}

How to parse an XML document in Go

Tagged encoding, parser, rss, golang, go, xml  Languages go

This example shows how to fetch and parse an XML feed with Go.

Save this in main_test.go:

package main

import (
    "bytes"
    "code.google.com/p/go-charset/charset"
    _ "code.google.com/p/go-charset/data" // Import charset configuration files
    "encoding/xml"
    "io/ioutil"
    "log"
    "net/http"
    "testing"
)

type RssFeed struct {
    XMLName xml.Name  `xml:"rss"`
    Items   []RssItem `xml:"channel>item"`
}

type RssItem struct {
    XMLName     xml.Name `xml:"item"`
    Title       string   `xml:"title"`
    Link        string   `xml:"link"`
    Description string   `xml:"description"`
    //NestedTag    string      xml:">nested>tags>"
}

func fetchURL(url string) []byte {
    resp, err := http.Get(url)
    if err != nil {
        log.Fatalf("unable to GET '%s': %s", url, err)
    }
    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        log.Fatalf("unable to read body '%s': %s", url, err)
    }
    return body
}

func parseXML(xmlDoc []byte, target interface{}) {
    reader := bytes.NewReader(xmlDoc)
    decoder := xml.NewDecoder(reader)
    // Fixes "xml: encoding \"windows-1252\" declared but Decoder.CharsetReader is nil"
    decoder.CharsetReader = charset.NewReader
    if err := decoder.Decode(target); err != nil {
        log.Fatalf("unable to parse XML '%s':\n%s", err, xmlDoc)
    }
}

func TestParseReport(t *testing.T) {
    var rssFeed = &RssFeed{}
    xmlDoc := fetchURL("https://news.ycombinator.com/rss")
    parseXML(xmlDoc, &rssFeed)
    for _, item := range rssFeed.Items {
        log.Printf("%s: %s", item.Title, item.Link)
    }
}

Run the code with go test.

Golang Worker Pattern Example

Tagged go, pattern, worker, channel, golang  Languages go

An example of the worker pattern in Go.

Executes, e.g., 20 units of work with 20 workers, taking 2 seconds each, in 2 seconds.

package main

//
// $ go build worker.go && WORKERS=20 JOBS=20 ./worker
//
import (
 "fmt"
 "os"
 "strconv"
 "sync"
 "time"
)

// A unit of work
type Rabbit struct {
 ID   int
 Name string
}

// The worker accepting a unit of work
type RabbitWork struct {
 Rabbit Rabbit
}

func main() {
 jobCount, _ :=     strconv.Atoi(os.Getenv("JOBS"))
 workerCount, _ :=  strconv.Atoi(os.Getenv("WORKERS"))
 start := time.Now()
 workers := make(chan RabbitWork, workers)
 var wg sync.WaitGroup

 // Add  workers to our startup "Icarus"
 for i := 0; i < workerCount; i++ {
  wg.Add(1)
  go func() {
   for work := range workers {
    // do some work on data
    var rabbit = work.Rabbit
    fmt.Println("Capturing", rabbit.Name, "(", rabbit.ID, ")")
    time.Sleep(2 * time.Second)
    fmt.Println("Skinned", rabbit.Name, "(", rabbit.ID, ")")
   }
   wg.Done()
  }()
 }

 // Give some work to the workers
 for i := 1; i <= jobCount; i++ {
  workers <- RabbitWork{
   Rabbit: Rabbit{Name: "Toffe", ID: i},
  }
 }

 // Wait for workers to do their job
 close(workers)
 wg.Wait()
 fmt.Println("Work took", time.Since(start).Seconds(), "seconds")
}

Notes

  1. WaitGroup waits for all work to finish
  2. wg.Wait() blocks until wg.Done() is called 20 times
  3. workers <- RabbitWork will block if all 20 workers are busy working
  4. for data := range tasks iterates over incoming work sent through the tasks channel
  5. close(workers) closes the channel
  6. 40 units of work with 20 workers will complete in 4 seconds
  7. also see: http://marcio.io/2015/07/handling-1-million-requests-per-minute-with-golang/

Checking if a row exists in Go (database/sql and SQLX)

Tagged golang, go, sqlx  Languages go

Code:

func rowExists(query string, args ...interface{}) bool {
    var exists bool
    query = fmt.Sprintf("SELECT exists (%s)", query)
    err := db.QueryRow(query, args...).Scan(&exists)
    if err != nil && err != sql.ErrNoRows {
            glog.Fatalf("error checking if row exists '%s' %v", args, err)
    }
    return exists
}

Usage:

if rowExists("SELECT id FROM feed_items WHERE url=$1", item.Link) {
    return
}

How to Join Two Tables with jmoiron/sqlx

Tagged go, sqlx, join, golang  Languages go

Here’s an example of how to define relationships between structs when using SQL joins and sqlx:

type Feed struct {
  ID        int       `json:"id"`
  Title     string    `json:"title"`
  URL       string    `json:"url"`
  CreatedAt time.Time `json:"created_at" db:"created_at"`
}

type FeedItem struct {
  ID          int            `json:"id"`
  Title       string         `json:"title"`
  Description sql.NullString `json:"description"`
  URL         string         `json:"url"`
  CreatedAt   time.Time      `json:"created_at" db:"created_at"`
  FeedId      int            `json:"feed_id" db:"feed_id"`
  Feed        `db:"feed"` // << NOTE PREFIX 
}

func getFeeds() (*[]Feed, error) {
  feeds := []Feed{}
  err := db.Select(&feeds, "SELECT * FROM feeds")
  return &feeds, err
}

func getFeedItems() (*[]FeedItem, error) {
  items := []FeedItem{}
  sql := `SELECT
      feed_items.*,
      feeds.id "feed.id",
      feeds.title "feed.title",
      feeds.url "feed.url",
      feeds.created_at "feed.created_at"
    FROM
      feed_items JOIN feeds ON feed_items.feed_id = feeds.id;`
  err := db.Select(&items, sql)
  return &items, err
}

Go and Postgres Example

Tagged polling, postgres, example, golang  Languages go, bash
package main

import (
    "database/sql"
    _ "github.com/lib/pq"
    "log"
    "time"
)

func pollChanges() {
    var id int
    var name string
    rows, err := db.Query("SELECT id, name FROM people")
    if err != nil {
        log.Fatal(err)
    }
    defer rows.Close()
    for rows.Next() {
        err := rows.Scan(&id, &name)
        if err != nil {
            log.Fatal(err)
        }
        log.Println(id, name)
    }
    err = rows.Err()
    if err != nil {
        log.Fatal(err)
    }
}

var db *sql.DB

func main() {
    var err error
    db, err = sql.Open("postgres", "")
    if err != nil {
        log.Fatal(err)
    }
    defer db.Close()
    for {
        time.Sleep(1000 * time.Millisecond)
        log.Println("Polling for changes...")
        pollChanges()
    }
}

Run:

PGUSER= PGPASSWORD= PGDATABASE=db_xyz PGSSLMODE=disable ./go-app

Notes

Using question marks instead of, e.g., $1 will result in the following error:

syntax error at or near ","
  • Scan requires a pointer. Forgetting to add an ampersand (Scan(id) vs Scan(&id)) will result in the following error:
Scan error on column index 0: destination not a pointer

Logging in Golang (including line numbers)

Tagged logging, debug, info, wrapper, line numbers, go, log  Languages bash, go

Features

  • Prints location of errors including line number
  • Debug mode

Code

var debug bool

func init() {
    debug = os.Getenv("DEBUG") != ""
}

// Info example:
//
// Info("timezone %s", timezone)
//
func Info(msg string, vars ...interface{}) {
    log.Printf(strings.Join([]string{"[INFO ]", msg}, " "), vars...)
}

// Debug example:
//
// Debug("timezone %s", timezone)
//
func Debug(msg string, vars ...interface{}) {
    if debug {
        log.Printf(strings.Join([]string{"[DEBUG]", msg}, " "), vars...)
    }
}

// Fatal example:
//
// Fatal(errors.New("db timezone must be UTC"))
//
func Fatal(err error) {
    pc, fn, line, _ := runtime.Caller(1)
    // Include function name if debugging
    if debug {
        log.Fatalf("[FATAL] %s [%s:%s:%d]", err, runtime.FuncForPC(pc).Name(), fn, line)
    } else {
        log.Fatalf("[FATAL] %s [%s:%d]", err, fn, line)
    }
}

// Error example:
//
// Error(errors.Errorf("Invalid timezone %s", timezone))
//
func Error(err error) {
    pc, fn, line, _ := runtime.Caller(1)
    // Include function name if debugging
    if debug {
        log.Printf("[ERROR] %s [%s:%s:%d]", err, runtime.FuncForPC(pc).Name(), fn, line)
    } else {
        log.Printf("[ERROR] %s [%s:%d]", err, fn, line)
    }
}

Example

2019/05/30 11:31:30 [FATAL] pq: unsupported sslmode "ddisable"; only "require" (default), "verify-full", "verify-ca", and "disable" supported [github.com/christianhellsten/goscheduler/poller.go:52]

XPath with namespaces example in Go

Tagged xpath, go, xml  Languages go
package main

import (
    "fmt"
    "github.com/moovweb/gokogiri"
    "github.com/moovweb/gokogiri/xpath"
    "io/ioutil"
)

func main() {
    xml, _ := ioutil.ReadFile("elrc-20130531.xml")
    doc, _ := gokogiri.ParseXml(xml)
    defer doc.Free()
    xp := doc.DocXPathCtx()
    xp.RegisterNamespace("xbrli", "http://www.xbrl.org/2003/instance")
    x := xpath.Compile("//xbrli:context")
    groups, err := doc.Search(x)
    if err != nil {
        fmt.Println(err)
    }
    for i, group := range groups {
        fmt.Println(i, group.Content())
    }
}
go get
go run main.go

Simple content negotation in Golang

Tagged accept, go  Languages go
func DoSomething(r *http.Request) string {
    accept := r.Header.Get("Accept")
    switch accept {
    case "application/json":
        return renderJSON()
    case "application/xml":
        return renderXML()
    }
    return renderHTML()
}