597 snippets – displaying 1–30

Validating HTML with validator.w3.org and Ruby

require "net/http"

def validate(html)
  response = Net::HTTP.start("validator.w3.org") do |r|
    query = "output=json&fragment=" + CGI.escape(html)
    r.post2("/check", query)
  end
  if "Invalid" == response["x-w3c-validator-status"]
    File.open('/tmp/html-validation.html', 'w') { |f| f << html }
    JSON.parse(response.body)
  end
end

# List all validation errors
html = File.read('/tmp/html-validation.html')
if errors = validate(html)
  puts errors
end

Validating HTML Emails with Nokogiri, Tidy, and validator.w3.org

# We're validating the HTML generated by the Devise mailer
user = User.new email: 'xxx@xxx'
mail = Devise.mailer.confirmation_instructions(user, self).deliver
html = mail.html_part.body.to_s

Validate with tidy:

File.open('/tmp/html-validation.html', 'w') { |f| f << html }
unless system "tidy -errors -q -f /tmp/validation-errors.txt /tmp/html-validation.html"
  fail "Validation failed:\n #{File.read('/tmp/validation-errors.txt')}"
end

Remember to install tidy first:

$ sudo apt-get install tidy

Validate with Nokogiri is not so fun or accurate:

# Validate XML
doc = Nokogiri.XML(html)
doc.errors

# Validate against a schema, if you want
xsd = Nokogiri::XML::Schema(open('http://www.w3.org/2002/08/xhtml/xhtml1-strict.xsd'))
xsd.validate(doc)

For validation with w3.org, see this script

How to show your public IP in your bash shell with PS1 and PROMPT_COMMAND variables

How to show your public IP address in your bash shell. Open ~/.bash_profile and add the following:

alias wanip='dig +short myip.opendns.com @resolver1.opendns.com'
# Uncomment to update public IP every time you press enter
# PROMPT_COMMAND="PS1=$(wanip)--'${PS1}'"
# Update public IP once at login
PS1="$(wanip)-$PS1"
# Also show public IP in the title bar
echo -ne "\033]0;$(wanip)-${USER}@${HOSTNAME}: \007"

Note that wanip blocks login until the command succeeds.

TODO:

Recursive Postgres Query That Finds Children of a Node in a Tree

The recursive query:

WITH RECURSIVE tree AS (
  SELECT id, name, parent_id FROM nodes WHERE id = 25
  UNION ALL
  SELECT nodes.id, nodes.name, nodes.parent_id FROM nodes, tree WHERE nodes.parent_id = tree.id)

This part can be seen as defining a virtual table called tree:

WITH RECURSIVE tree

The part finds the root node and initializes the recursive function:

SELECT id, name, parent_id FROM nodes WHERE id = 25

The next part fetches children recursively:

SELECT nodes.id, nodes.name, nodes.parent_id FROM nodes, tree WHERE nodes.parent_id = tree.id)

This SQL query shows the descendants, and the parent which has an ID of 25:

SELECT * FROM tree;

In Ruby you could use this code to generate your SQL:

  def tree_sql(opts={})
    table = opts.fetch(:table) # e.g. 'categories'
    cols = opts.fetch(:cols) # e.g. %w(id name)
    <<-SQL
      WITH RECURSIVE #{table}_tree AS (
        -- initialize
        SELECT
          #{cols.join(', ')}, 0 as n_depth
        FROM
          #{table}
        WHERE
          -- Use bind variables and ? here if you want
          id = #{parent_id}
        UNION ALL
          -- iterate
          SELECT
            #{cols.map { |col| "#{table}.#{col}" }.join(', ')}, n_depth +1
          FROM
            #{table}, #{table}_tree
          WHERE
            #{table}.parent_id = #{table}_tree.id)
    SQL
  end

Details:

  • Use UNION instead of UNION ALL if you have duplicates, or if you’re not worried about performance when Postgres has to scan for duplicates.
  • WITH Queries in Postgres
  • Recursive queries can loop indefinitely. If this happens, use the ANY functions.

Frozen ActiveRecord Attributes Validator

Disallows update of frozen attributes:

#
# Validates that an attribute cannot be changed after object has been created.
#
# Usage:
#   validates :user_id, :token, frozen_attribute: true, allow_blank: true
#
# allow_blank: set to true to allow setting nil attributes
#
class FrozenAttributeValidator < ActiveModel::EachValidator
  def validate_each(record, attribute, value)
    changed = record.send("#{attribute}_changed?")
    was_not_blank = if options[:allow_blank]
                      !(record.send("#{attribute}_was").blank? && value.present?)
                    else
                      true
                    end
    if record.persisted? && changed && was_not_blank
      record.errors.add(attribute, 'is frozen')
    end
    true
  end
end

Put the thing in app/validators/frozen_attribute_validator.rb. Use it like this:

class Horse < ActiveRecord::Base
  validates :user_id, :token, frozen_attribute: true, allow_blank: true

Deployment Tasks for Puma and Mina (Start, Restart, Stop, Phased Restart)

minapumachristianVersion 2

def puma_conf
  @puma_conf ||= begin
    require 'puma/configuration'
    config = Puma::Configuration.new(config_file: "config/puma.rb.#{rails_env}")
    config.load
    OpenStruct.new(config.options)
  end
end

set :puma_cmd, 'bundle exec puma'
set :pumactl_cmd, 'bundle exec pumactl'
set :puma_state_path, puma_conf.state
set :puma_start_options, "-q -d -e #{rails_env}  -C config/puma.rb.#{rails_env}"

namespace :puma do
  desc 'Start puma'
  task :start => :environment do
    queue "cd #{full_current_path} && #{puma_cmd} #{puma_start_options}"
  end

  desc 'Stop puma'
  task :stop => :environment do
    queue "cd #{full_current_path} && #{pumactl_cmd} -S #{puma_state_path} stop"
  end

  desc 'Restart puma'
  task :restart => :environment do
    begin
      queue "cd #{full_current_path} && #{pumactl_cmd} -S #{puma_state_path} restart"
    rescue => ex
      puts "Failed to restart puma: #{ex}\nAssuming not started."
    end
  end

  desc 'Restart puma (phased restart)'
  task :phased_restart => :environment do
    run "cd #{full_current_path} && #{pumactl_cmd} -S #{puma_state_path} phased-restart"
  end
end

Creating A Responsive Mobile-First Template with Susy

Susy version 1:

// http://susy.oddbird.net/tutorial/
@import "susy/sass/_susy"
@include border-box-sizing

$show-grid-backgrounds: true

// Mobile-first grid
$total-cols             : 4 
$col-width              : 4em
$gutter-width           : 1em
$side-gutter-width      : $gutter-width
$container-style        : static

// Tablet grid
$tablet-cols	  : 10
// 768px
$tablet-width	  : 48em
$tablet         : $tablet-width $tablet-cols
 
$desktop-cols   : 12
// 960px
$desktop-width	: 60em
$desktop        : $desktop-width $desktop-cols

.container
  // Tell susy we have 3 different layouts
  @include container($total-cols, $tablet-cols, $desktop-cols)
  @include susy-grid-background

  // Mobile phones
  #content 
    @include span-columns(4, 4)
  #sidebar 
    @include span-columns(4, 4)

  // Tablets
  @include at-breakpoint($tablet)
    @include container

    #content 
      @include span-columns(7, $tablet-cols)
    #sidebar 
      @include span-columns(3 omega, $tablet-cols)

  // Desktops
  @include at-breakpoint($desktop)
    @include container
    #content 
      @include span-columns(8, $desktop-cols)
    #sidebar 
      @include span-columns(4 omega, $desktop-cols)

Susy version 2????

ElasticSearch Wildcard and NGram Search With Tire

How to implement wildcard search with Tire and Elasticsearch:

  settings analysis: {
    filter: {
      ngram_filter: {
        type: "nGram",
        min_gram: 1,
        max_gram: 15
      }
    },
    analyzer: {
      index_ngram_analyzer: {
        tokenizer: "standard",
        filter: ['standard', 'lowercase', "stop", "ngram_filter"],
        type: "custom"
      },
      search_ngram_analyzer: {
        tokenizer: "standard",
        filter: ['standard', 'lowercase', "stop"],
        type: "custom"
      }
    }
  }

  mapping do
    indexes :name,
      search_analyzer: 'search_ngram_analyzer',
      index_analyzer: 'index_ngram_analyzer', 
      #analyzer: 'index_ngram_analyzer', 
      boost: 100.0
      #
  end

With curl, make sure the mapping is set up properly:

curl 'http://localhost:9200/activities/_mapping?pretty=true'
{
  "skulls" : {
    "skull" : {
      "_all" : {
        "auto_boost" : true
      },
      "properties" : {
        "name" : {
          "type" : "string",
          "boost" : 100.0,
          "analyzer" : "index_ngram_analyzer"
        }
      }
    }
  }
}

You now have wildcard search as long as you remember to specify the fields that you want to search, because by default the _all field is used for search:

# This searches the _all field
curl 'http://localhost:9200/activities/_search?q=simpsons&pretty=true'

# Yes, it really works
curl -XGET 'http://localhost:9200/activities/_search?pretty' -d ' 
{ 
   "query" : { 
      "query_string" : { 
         "query" : "simpsons", 
         "fields" : ["name"] 
      } 
   } 
}' 

Fontello script

Easily download and install Fontello font icons with this makefile; no more manual downloading and copying:

#
# Easily download and install Fontello font icons.
#
# Original version:
# https://gist.github.com/puzrin/5537065
#
 
# Note this directory will be deleted
TMP_DIR      ?= ./tmp/fontello
# The Fontello config.json file
CONF_FILE    ?= ./config/fontello.json
# Where do you want the font files to be?
FONT_DIR     ?= ./public/font
# Where do you want the CSS files to be?
CSS_DIR      ?= ./app/css/vendor
 
### Don't edit below ###
 
FONTELLO_HOST ?= http://fontello.com
 
download:
	@if test ! `which curl` ; then \
		echo 'Install curl first.' >&2 ; \
		exit 128 ; \
		fi
	curl --silent --show-error --fail --output .fontello \
		--form "config=@${CONF_FILE}" \
		${FONTELLO_HOST}
	open ${FONTELLO_HOST}/`cat .fontello`

install:
	@if test ! `which unzip` ; then \
		echo 'Install unzip first.' >&2 ; \
		exit 128 ; \
		fi
	@if test ! -e .fontello ; then \
		echo 'Run `make fontopen` first.' >&2 ; \
		exit 128 ; \
		fi
	rm -rf ${TMP_DIR} .fontello.zip
	curl --silent --show-error --fail --output .fontello.zip \
		${FONTELLO_HOST}/`cat .fontello`/get
	unzip .fontello.zip -d ${TMP_DIR}
	mkdir -p ${FONT_DIR}
	mkdir -p ${CSS_DIR}
	cp -R ${TMP_DIR}/*/font/* ${FONT_DIR}
	cp -R ${TMP_DIR}/*/css/* ${CSS_DIR}
	rm -rf ${TMP_DIR} .fontello.zip

Fix for "undefined method `encoding' for nil:NilClass"

This code:

img src="/xyz/#{@house.address}"

Caused this:

undefined method `encoding' for nil:NilClass

/usr/local/rbenv/versions/2.0.0-p247/lib/ruby/2.0.0/cgi/util.rb in escape
    encoding = string.encoding

Why? Because address is nil and Slim tried to escape nil with CGI.escape, which doesn’t work. The error was reported to be on line x when the error was on line y…

How to use Vagrant, Chef and Librarian to set up and configure a development environment

Save this as Vagrantfile:

# Instructions
# -----------------
# Follow these instructions to set up a new development environment.
#
# Setup tools
# -----------------
# gem install vagrant
#
# # librarian is chef's bundler
# gem install librarian-chef
#
# Install Chef cookbooks
# -----------------
# librarian-chef install
#
# Install VM
# -----------------
# vagrant up
#
# Provision VM, i.e install software with Chef
# -----------------
# vagrant provision
#
# Setup SSH
# -----------------
# vagrant ssh-config >> ~/.ssh/config
#
# sudo echo "10.11.12.13 development" >> /etc/hosts
#
# SSH into your dev environment
# -----------------
# ssh development

Vagrant::Config.run do |config|
 # Configure IP, etc
  config.vm.network :hostonly, "10.11.12.13"
  config.vm.host_name = "development"
  config.vm.box = "precise64"
  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
  # Use 1GB RAM
  config.vm.customize ["modifyvm", :id, "--memory", 1024] 
  # Link ~/projects on local machine to ~/projects on VM
  config.vm.share_folder "projects", "/home/vagrant/projects", "~/projects", :nfs => true
  config.vm.customize ["setextradata", :id, "VBoxInternal2/SharedFoldersEnableSymlinksCreate/", "1"]
  # Forward ports
  config.vm.forward_port 80, 80
  config.vm.forward_port 6080, 6080
  config.vm.forward_port 8080, 8080
  config.vm.forward_port 3000, 3000
  config.vm.forward_port 9200, 9200
  config.vm.forward_port 8888, 8888

  config.vm.provision :chef_solo, :log_level => :debug do |chef|
    chef.cookbooks_path = "cookbooks"
    #chef.cookbooks_path = ["cookbooks", "vendor/cookbooks"]
    chef.add_recipe "apt"
    chef.add_recipe "vim"

    chef.add_recipe "build-essential"
    chef.add_recipe "git"
    chef.add_recipe "curl"
    chef.add_recipe "curl::devel"
    chef.add_recipe "java"
    chef.add_recipe "ruby_build"
    chef.add_recipe "rbenv::system"
    chef.add_recipe "rbenv::vagrant"
    chef.add_recipe "sqlite"

    chef.add_recipe "nodejs"
    chef.add_recipe "percona::server"
    chef.add_recipe "percona::client"
    chef.add_recipe "elasticsearch"

    # Run rubygems-bundler
    config.vm.provision :shell, :inline => "gem regenerate_binstubs"

    # To install APT packages use:
    #%w(libmysql18-ruby libmysqlclient18-dev).each do |package|
      #config.vm.provision :shell, :inline => "sudo aptitude -y install #{package}"
    #end

    chef.json = { 
      "rbenv" => {
        "rubies" => [ "1.9.3-p448", "2.0.0-p247" ],
        "global" => "2.0.0-p247",
        "gems" => {
          "2.0.0-p247" => [
            { :name => "bundler" }, { :name => "rubygems-bundler" }
          ],
          "1.9.3-p448" => [
            { :name => "bundler" },{ :name => "rubygems-bundler" }
          ]
        }
      }
    }
  end
end

Save the following in Cheffile:

site "http://community.opscode.com/api/v1"

cookbook "percona"

cookbook "rbenv",
  :git => "https://github.com/fnichol/chef-rbenv",
  :ref => "v0.7.2"

cookbook "percona"
cookbook "git"
cookbook "nodejs"
cookbook "ruby_build"
cookbook "sqlite"
cookbook "elasticsearch"
cookbook "curl", git: 'https://github.com/phlipper/chef-curl'
cookbook "build-essential"
cookbook "vim"
cookbook "java"

Read the instructions in Vagrantfile…

You should now be able to set up a virtual machine that can be used for web development in as long as it takes to compile and install all the software. No extra configuration is needed…

This VM will have software, including:

  • Ubuntu 12.04 Precise, see vagrantbox.es for other Linux versions
  • Ruby 2 and 1.9 with rbenv
  • Elasticsearch
  • Percona MySQL
  • git and some other tools

Rails find_or_initialize and find_or_create methods are deprecated

The find_or_initialize and find_or_create methods have been deprecated in Rails x.×.×. This is what you need to do:

Before:
User.find_or_create_by_name('XXX')

After:
User.where(name: 'XXX').first_or_create
User.where(name: 'XXX').first_or_create!

Before:
User.find_or_initialize_by_name('XXX')

After:
User.where(name: 'XXX').first_or_initialize

If you want the old methods, see https://github.com/rails/activerecord-deprecated_finders.

Also see http://apidock.com/rails/ActiveRecord/Relation/first_or_create

Installing Scrapyd on Debian

Install Debian pre-requisites

sudo apt-get install python-pip python-dev
# Scrapyd image module requirements
sudo apt-get install libjpeg-dev libfreetype6-dev zlib1g-dev libpng12-dev

Install Python libraries

Customize for your project:

sudo pip install unidecode mysql-python pyes pil

Use pip freeze if you want:

pip freeze > requirements.txt

Install scrapy

git clone https://github.com/scrapy/scrapy.git
cd scrapy
sudo python setup.py install

Install scrapyd

git clone https://github.com/scrapy/scrapyd.git
cd scrapyd
sudo python setup.py install

Deploy project to Scrapyd (http://localhost:6800)

cd project
scrapy deploy

Run

- Start a spider
curl http://localhost:6800/schedule.json -d project=default -d spider=xxx

- List spiders
curl http://localhost:6800/listspiders.json?project=default

- List jobs
curl http://localhost:6800/listjobs.json?project=default

- Cancel job
curl http://localhost:6800/cancel.json -d project=default -d job=4abb6c78fd1a11e28ed9fefdb24fae0a

ZeroClipboard example on how to use multiple copy buttons and load data through Ajax

$ ->
  $('.copy-button').click (e) ->
    e.preventDefault()

  clip = new ZeroClipboard $('.copy-button'),
    moviePath: "/swf/ZeroClipboard.swf"

  clip.on 'mousedown', (client) ->
    link = $(this)
    $.ajax
      url: link.attr("href") + ".json"
      success: (content) ->
        console.debug('Copied to clipboard: ' + content)
        clip.setText content
      async: false

  clip.on 'complete', (client, args) ->
    link = $(this)
    link.find('.text').html('Copied&hellip;')

How to Convert SHP to KML with GDAL

After reading Google’s advice on the matter we add the following to ~/.bash_profile (see GDAL ReadMe.rtf for details):

export PATH=/Library/Frameworks/GDAL.framework/Programs:$PATH

We can now convert SHP file to KML:

ogr2ogr -f “KML" -where “qs_adm0_a3=’FIN’" ~/Downloads/xxx.kml ~/Downloads/qs_adm1_region/qs_adm1_region.shp

ogr2ogr -f “KML" -where “qs_adm0=’Finland’" ~/Downloads/xxx.kml ~/Downloads/qs_adm1_region/qs_adm1_region.shp

ogr2ogr -f “KML" -where “qs_adm0=’Finland’ and qs_level=’adm1_region’" ~/Downloads/xxx.kml ~/Downloads/qs_adm1_region/qs_adm1_region.shp

ogr2ogr -f “KML" ~/Downloads/all.kml ~/Downloads/qs_adm1_region/qs_adm1_region.shp

Notice the “where” query. It pulls out data from the XML file where the SimpleData tag matches your query:

<SimpleData name="qs_adm0_a3">BEL</SimpleData>

Render the data with with Python, TileMill, or QuantumGis

How to use ElasticSearch with Python

This is a short example on how to use ElasticSearch with Python.

First install pyes (pyes documentation).

Then run this code:

# https://pyes.readthedocs.org/en/latest/references/pyes.es.html
# http://davedash.com/2011/02/25/bulk-load-elasticsearch-using-pyes/
from pyes import *

index_name = 'xxx'
type_name = 'car'

conn = ES('127.0.0.1:9200', timeout=3.5)

docs = [
    {"name":"good",  "id":'1'},
    {"name":"bad", "id":'2'},
    {"name":"ugly", "id":'3'}
]

# Bulk index
for doc in docs:
    # index(doc, index, doc_type, id=None, parent=None, force_insert=False, op_type=None, bulk=False, version=None, querystring_args=None)
    conn.index(doc, index_name, type_name, id=doc['id'], bulk=True)

print conn.refresh()

# Search
def search(query):
    q = StringQuery(query, default_operator="AND")
    result = conn.search(query=q, indices=[index_name])
    for r in result:
        print r


search("good")

You can also use CURL to verify that it works:

# Show index mapping
curl -vvv "http://127.0.0.1:9200/xxx/_mapping?pretty=1"

# Delete index
curl -XDELETE -vvv "http://127.0.0.1:9200/xxx"

# Search
curl -vvv "http://127.0.0.1:9200/xxx/_search?pretty=1"

Levenshtein distance for MySQL

Levenshtein distance for MySQL:

DELIMITER $$
CREATE FUNCTION levenshtein( s1 VARCHAR(255), s2 VARCHAR(255) ) 
  RETURNS INT 
  DETERMINISTIC 
  BEGIN 
    DECLARE s1_len, s2_len, i, j, c, c_temp, cost INT; 
    DECLARE s1_char CHAR; 
    -- max strlen=255 
    DECLARE cv0, cv1 VARBINARY(256); 
    SET s1_len = CHAR_LENGTH(s1), s2_len = CHAR_LENGTH(s2), cv1 = 0x00, j = 1, i = 1, c = 0; 
    IF s1 = s2 THEN 
      RETURN 0; 
    ELSEIF s1_len = 0 THEN 
      RETURN s2_len; 
    ELSEIF s2_len = 0 THEN 
      RETURN s1_len; 
    ELSE 
      WHILE j <= s2_len DO 
        SET cv1 = CONCAT(cv1, UNHEX(HEX(j))), j = j + 1; 
      END WHILE; 
      WHILE i <= s1_len DO 
        SET s1_char = SUBSTRING(s1, i, 1), c = i, cv0 = UNHEX(HEX(i)), j = 1; 
        WHILE j <= s2_len DO 
          SET c = c + 1; 
          IF s1_char = SUBSTRING(s2, j, 1) THEN  
            SET cost = 0; ELSE SET cost = 1; 
          END IF; 
          SET c_temp = CONV(HEX(SUBSTRING(cv1, j, 1)), 16, 10) + cost; 
          IF c > c_temp THEN SET c = c_temp; END IF; 
            SET c_temp = CONV(HEX(SUBSTRING(cv1, j+1, 1)), 16, 10) + 1; 
            IF c > c_temp THEN  
              SET c = c_temp;  
            END IF; 
            SET cv0 = CONCAT(cv0, UNHEX(HEX(c))), j = j + 1; 
        END WHILE; 
        SET cv1 = cv0, i = i + 1; 
      END WHILE; 
    END IF; 
    RETURN c; 
  END$$


CREATE FUNCTION levenshtein_ratio( s1 VARCHAR(255), s2 VARCHAR(255) ) 
  RETURNS INT 
  DETERMINISTIC 
  BEGIN 
    DECLARE s1_len, s2_len, max_len INT; 
    SET s1_len = LENGTH(s1), s2_len = LENGTH(s2); 
    IF s1_len > s2_len THEN  
      SET max_len = s1_len;  
    ELSE  
      SET max_len = s2_len;  
    END IF; 
    RETURN ROUND((1 - LEVENSHTEIN(s1, s2) / max_len) * 100); 
  END$$

DELIMITER ;

Also see this.

Now you can run these queries:

select levenshtein('butt', 'but') from test;
select levenshtein_ratio('butt', 'but') from test;