How to implement to_json in Crystal lang

Tagged crystal, to_json  Languages crystal

This is an example of how to implement a generic to_json method in Crystal lang:

class Document
  def initialize
    @name = "xyz"
    ...
  end
  
  # Returns an array containing all instance variables. Implemented with a macro.
  def instance_vars
    {{
      @type.instance_vars.map do |var|
        [ var.name.stringify, var.id ]
      end
    }}
  end

  # Returns a JSON string containing all instance variables
  def to_json
    result = String.build do |io|
      JSON::PrettyWriter.new(io, indent: "  ").json_object do |object|
        instance_vars.each do |var|
          name, value = var
          case value
          when XML::Node
            # skip
          else
            object.field name, value
          end
        end
      end
    end
  end

Hello Postgres and REST in Crystal

Tagged crystal, postgres, rest  Languages yaml, crystal, bash

Install Crystal

brew install crystal-lang
mkdir projects/hello
shards init

Set up dependencies

dependencies:
  pg:
    github: will/crystal-pg
    version: "~> 0.5"
  kemal:
    github: sdogruyol/kemal
    version: "~> 0.16.1"

Write a simple REST API

require "kemal"
require "json"
require "pg"

PG_URL = "postgres://postgres@localhost:5432/xxx"
DB     = PG.connect PG_URL

get "/" do |env|
  env.response.content_type = "application/json"
  users = DB.exec("SELECT * FROM users")
  users.to_hash.map do |user|
    {first_name: user["first_name"].as(String), last_name: user["last_name"].as(String)}
  end.to_json
end

Kemal.run(4567)

Storing, querying, and indexing XML with Postgres

Tagged postgres, xbrl, xml, xpath  Languages xml, sql

Create table

CREATE TABLE xbrl_reports
(
  id serial primary key NOT NULL,
  doc xml NOT NULL,
  cik varchar(255) NOT NULL
);

Create function for importing XML

-- http://tapoueh.org/blog/2009/02/05-importing-xml-content-from-file
create or replace function xml_import(filename text)
  returns xml
  volatile
  language plpgsql as
$f$
    declare
        content bytea;
        loid oid;
        lfd integer;
        lsize integer;
    begin
        loid := lo_import(filename);
        lfd := lo_open(loid,262144);
        lsize := lo_lseek(lfd,0,2);
        perform lo_lseek(lfd,0,0);
        content := loread(lfd,lsize);
        perform lo_close(lfd);
        perform lo_unlink(loid);
 
        return xmlparse(document convert_from(content,'UTF8'));
    end;
$f$;

Import XML

-- Import XML file into Postgres
insert into xbrl_reports(doc, cik) values(xml_import('/Users/Christian/Downloads/2016q3/ibm-20160930.xml'), '1');

XML namespaces:

<xbrl
  xmlns="http://www.xbrl.org/2003/instance"
  xmlns:dei="http://xbrl.sec.gov/dei/2014-01-31"

Query data

-- Check if dei::TradingSymbol exists => t
SELECT xpath('//xbrl:xbrl/dei:TradingSymbol/text()', doc, '{{xbrl,http://www.xbrl.org/2003/instance},{dei,http://xbrl.sec.gov/dei/2014-01-31}}') from xbrl_reports;

-- Extract dei:TradingSymbol by declaring dei namespace => {IBM}
SELECT xpath('//xbrl:xbrl/dei:TradingSymbol/text()', doc, '{{xbrl,http://www.xbrl.org/2003/instance},{dei,http://xbrl.sec.gov/dei/2014-01-31}}') from xbrl_reports;

-- Extract dei:TradingSymbol by adding ((...)[1]::text) => IBM
SELECT ((xpath('//xbrl:xbrl/dei:TradingSymbol/text()', doc, '{{xbrl,http://www.xbrl.org/2003/instance},{dei,http://xbrl.sec.gov/dei/2014-01-31}}'))[1]::text) from xbrl_reports;

Index data

-- Create index for faster lookups
create index xbrl_reports_ticker_idx on xbrl_reports using btree ((( xpath('//xbrl:xbrl/dei:TradingSymbol/text()', doc, '{{xbrl,http://www.xbrl.org/2003/instance},{dei,http://xbrl.sec.gov/dei/2014-01-31}}') )[1]::text)); 

Materialized views for performance

CREATE MATERIALIZED VIEW company_reports AS SELECT
  ((xpath('//xbrl:xbrl/dei:TradingSymbol/text()', doc, '{{xbrl,http://www.xbrl.org/2003/instance},{dei,http://xbrl.sec.gov/dei/2014-01-31}}'))[1]::text) as ticker,
  ((xpath('//xbrl:xbrl/dei:EntityRegistrantName/text()', doc, '{{xbrl,http://www.xbrl.org/2003/instance},{dei,http://xbrl.sec.gov/dei/2014-01-31}}'))[1]::text) as name,
  ((xpath('//xbrl:xbrl/dei:DocumentType/text()', doc, '{{xbrl,http://www.xbrl.org/2003/instance},{dei,http://xbrl.sec.gov/dei/2014-01-31}}'))[1]::text) as document_type,
  ((xpath('//xbrl:xbrl/dei:DocumentPeriodEndDate/text()', doc, '{{xbrl,http://www.xbrl.org/2003/instance},{dei,http://xbrl.sec.gov/dei/2014-01-31}}'))[1]::text) as quarter,
  ((xpath('//xbrl:xbrl/dei:EntityCommonStockSharesOutstanding/text()', doc, '{{xbrl,http://www.xbrl.org/2003/instance},{dei,http://xbrl.sec.gov/dei/2014-01-31}}'))[1]::text) as shares_outstanding
FROM xbrl_reports;

A simple log wrapper for Go

Tagged go, log, logging  Languages go

Version 1

This is a simple log wrapper for go’s built in log package. Put the following code in a file named log.go:

package main

import (
    golog "log"
    "os"
)

//
// Use ioutil.Discard instead of f to send log to '/dev/null'
//
type Logger struct {
    Info     *golog.Logger
    Error    *golog.Logger
    Debug    *golog.Logger
    FatalLog *golog.Logger
}
func (logger Logger) Fatal(format string, args ...interface{}) {
    logger.FatalLog.Printf(format, args...)
    golog.Fatalf(format, args...)
}

var logFile *os.File
var log Logger

func init() {
    fileName := os.Getenv("LOG")
    if len(fileName) > 0 {
        var err error
        logFile, err = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
        if err != nil {
            golog.Fatalf("error opening file: %v", err)
        }
    } else {
        logFile = os.Stdout
    }
    log.Info = golog.New(logFile, "INFO  ", golog.LstdFlags)
    log.Error = golog.New(logFile, "ERROR ", golog.LstdFlags)
    log.Debug = golog.New(logFile, "DEBUG ", golog.LstdFlags)
    log.FatalLog = golog.New(logFile, "FATAL ", golog.LstdFlags)
}

Example:

log.Info.Printf("Notifications %v", notifications)
log.Debug.Printf("Notifications %v", notifications)

Version 2

To remove the call to Printf you could do this:

type Log struct {
...
    //Info     *golog.Logger
    Info     func(string, ...interface{})
...
}

func init() {
...
    log.Info = golog.New(logFile, "INFO  ", golog.LstdFlags).Printf
...
}

Now you can simply call log.Info:

log.Info("Notifications %v", notifications)
log.Debug("Notifications %v", notifications)

See https://golang.org/pkg/log/#Logger

Mattermost database schema and design

Tagged communication, database, mattermost, schema, slack  Languages sql, bash

This review was done on Mattermost version 3.4.0.

Install mattermost

docker run --name mattermost-preview -d --publish 8065:8065 mattermost/mattermost-preview
docker exec -ti mattermost-preview /bin/bash
...
mysql -u root -p
use mattermost_test

Database password is mostest.

Schema

mysql> describe Posts;
+------------+--------------+------+-----+---------+-------+
| Field      | Type         | Null | Key | Default | Extra |
+------------+--------------+------+-----+---------+-------+
| Id         | varchar(26)  | NO   | PRI | NULL    |       |
| CreateAt   | bigint(20)   | YES  | MUL | NULL    |       |
| UpdateAt   | bigint(20)   | YES  | MUL | NULL    |       |
| DeleteAt   | bigint(20)   | YES  |     | NULL    |       |
| UserId     | varchar(26)  | YES  |     | NULL    |       |
| ChannelId  | varchar(26)  | YES  | MUL | NULL    |       |
| RootId     | varchar(26)  | YES  | MUL | NULL    |       |
| ParentId   | varchar(26)  | YES  |     | NULL    |       |
| OriginalId | varchar(255) | YES  |     | NULL    |       |
| Message    | text         | YES  | MUL | NULL    |       |
| Type       | varchar(26)  | YES  |     | NULL    |       |
| Props      | text         | YES  |     | NULL    |       |
| Hashtags   | text         | YES  | MUL | NULL    |       |
| Filenames  | text         | YES  |     | NULL    |       |
+------------+--------------+------+-----+---------+-------+
14 rows in set (0.00 sec)

mysql> describe Channels;
+---------------+--------------+------+-----+---------+-------+
| Field         | Type         | Null | Key | Default | Extra |
+---------------+--------------+------+-----+---------+-------+
| Id            | varchar(26)  | NO   | PRI | NULL    |       |
| CreateAt      | bigint(20)   | YES  |     | NULL    |       |
| UpdateAt      | bigint(20)   | YES  |     | NULL    |       |
| DeleteAt      | bigint(20)   | YES  |     | NULL    |       |
| TeamId        | varchar(26)  | YES  | MUL | NULL    |       |
| Type          | varchar(1)   | YES  |     | NULL    |       |
| DisplayName   | varchar(64)  | YES  |     | NULL    |       |
| Name          | varchar(64)  | YES  | MUL | NULL    |       |
| Header        | text         | YES  |     | NULL    |       |
| Purpose       | varchar(128) | YES  |     | NULL    |       |
| LastPostAt    | bigint(20)   | YES  |     | NULL    |       |
| TotalMsgCount | bigint(20)   | YES  |     | NULL    |       |
| ExtraUpdateAt | bigint(20)   | YES  |     | NULL    |       |
| CreatorId     | varchar(26)  | YES  |     | NULL    |       |
+---------------+--------------+------+-----+---------+-------+


mysql> describe ChannelMembers;
+--------------+-------------+------+-----+---------+-------+
| Field        | Type        | Null | Key | Default | Extra |
+--------------+-------------+------+-----+---------+-------+
| ChannelId    | varchar(26) | NO   | PRI | NULL    |       |
| UserId       | varchar(26) | NO   | PRI | NULL    |       |
| Roles        | varchar(64) | YES  |     | NULL    |       |
| LastViewedAt | bigint(20)  | YES  |     | NULL    |       |
| MsgCount     | bigint(20)  | YES  |     | NULL    |       |
| MentionCount | bigint(20)  | YES  |     | NULL    |       |
| NotifyProps  | text        | YES  |     | NULL    |       |
| LastUpdateAt | bigint(20)  | YES  |     | NULL    |       |
+--------------+-------------+------+-----+---------+-------+
8 rows in set (0.00 sec)


mysql> describe Teams;
+-----------------+--------------+------+-----+---------+-------+
| Field           | Type         | Null | Key | Default | Extra |
+-----------------+--------------+------+-----+---------+-------+
| Id              | varchar(26)  | NO   | PRI | NULL    |       |
| CreateAt        | bigint(20)   | YES  |     | NULL    |       |
| UpdateAt        | bigint(20)   | YES  |     | NULL    |       |
| DeleteAt        | bigint(20)   | YES  |     | NULL    |       |
| DisplayName     | varchar(64)  | YES  |     | NULL    |       |
| Name            | varchar(64)  | YES  | UNI | NULL    |       |
| Email           | varchar(128) | YES  |     | NULL    |       |
| Type            | varchar(255) | YES  |     | NULL    |       |
| CompanyName     | varchar(64)  | YES  |     | NULL    |       |
| AllowedDomains  | text         | YES  |     | NULL    |       |
| InviteId        | varchar(32)  | YES  | MUL | NULL    |       |
| AllowOpenInvite | tinyint(1)   | YES  |     | NULL    |       |
+-----------------+--------------+------+-----+---------+-------+

mysql> describe TeamMembers;
+----------+-------------+------+-----+---------+-------+
| Field    | Type        | Null | Key | Default | Extra |
+----------+-------------+------+-----+---------+-------+
| TeamId   | varchar(26) | NO   | PRI | NULL    |       |
| UserId   | varchar(26) | NO   | PRI | NULL    |       |
| Roles    | varchar(64) | YES  |     | NULL    |       |
| DeleteAt | bigint(20)  | YES  |     | NULL    |       |
+----------+-------------+------+-----+---------+-------+
4 rows in set (0.00 sec)


mysql> describe Users;
+--------------------+--------------+------+-----+---------+-------+
| Field              | Type         | Null | Key | Default | Extra |
+--------------------+--------------+------+-----+---------+-------+
| Id                 | varchar(26)  | NO   | PRI | NULL    |       |
| CreateAt           | bigint(20)   | YES  |     | NULL    |       |
| UpdateAt           | bigint(20)   | YES  |     | NULL    |       |
| DeleteAt           | bigint(20)   | YES  |     | NULL    |       |
| Username           | varchar(64)  | YES  | UNI | NULL    |       |
| Password           | varchar(128) | YES  |     | NULL    |       |
| AuthData           | varchar(128) | YES  | UNI | NULL    |       |
| AuthService        | varchar(32)  | YES  |     | NULL    |       |
| Email              | varchar(128) | YES  | UNI | NULL    |       |
| EmailVerified      | tinyint(1)   | YES  |     | NULL    |       |
| Nickname           | varchar(64)  | YES  |     | NULL    |       |
| FirstName          | varchar(64)  | YES  |     | NULL    |       |
| LastName           | varchar(64)  | YES  |     | NULL    |       |
| Roles              | varchar(64)  | YES  |     | NULL    |       |
| AllowMarketing     | tinyint(1)   | YES  |     | NULL    |       |
| Props              | text         | YES  |     | NULL    |       |
| NotifyProps        | text         | YES  |     | NULL    |       |
| LastPasswordUpdate | bigint(20)   | YES  |     | NULL    |       |
| LastPictureUpdate  | bigint(20)   | YES  |     | NULL    |       |
| FailedAttempts     | int(11)      | YES  |     | NULL    |       |
| Locale             | varchar(5)   | YES  |     | NULL    |       |
| MfaActive          | tinyint(1)   | YES  |     | NULL    |       |
| MfaSecret          | varchar(128) | YES  |     | NULL    |       |
+--------------------+--------------+------+-----+---------+-------+
23 rows in set (0.00 sec)

Data

mysql> select * from Channels;
+----------------------------+---------------+---------------+----------+----------------------------+------+-------------+--------------------------------------------------------+--------+---------+---------------+---------------+---------------+----------------------------+
| Id                         | CreateAt      | UpdateAt      | DeleteAt | TeamId                     | Type | DisplayName | Name                                                   | Header | Purpose | LastPostAt    | TotalMsgCount | ExtraUpdateAt | CreatorId                  |
+----------------------------+---------------+---------------+----------+----------------------------+------+-------------+--------------------------------------------------------+--------+---------+---------------+---------------+---------------+----------------------------+
| 1h3rs83b3bnwzn3xb1ow49rssy | 1474786289480 | 1474786289480 |        0 | chjgwwy9qbbsdcz4ry4a6fbsoy | O    | Town Square | town-square                                            |        |         | 1474786555810 |             2 | 1474786555808 |                            |
| 1muwbod1hjrmzcr6iukhzqk4dc | 1474786289482 | 1474786289482 |        0 | chjgwwy9qbbsdcz4ry4a6fbsoy | O    | Off-Topic   | off-topic                                              |        |         | 1474786565693 |             2 | 1474786555815 |                            |
| 3pocxn4za7rd58q6rzcycgerna | 1474786510558 | 1474786510558 |        0 | ehwmnuwmztrujxhtb65kpgq4sy | O    | Off-Topic   | off-topic                                              |        |         | 1474786524398 |             1 | 1474786510570 |                            |
| f3j13bk4fi8qb8hh4mzgfbmmjc | 1474786769364 | 1474786769364 |        0 |                            | D    |             | abkob7omkjynfqfsc8shkbwkfo__u7s1oanujifg8ywuirnmhk8r3a |        |         | 1474786777955 |             1 | 1474786769364 |                            |
| g46cuaw3u781bjfyowudtfso7y | 1474786510553 | 1474786510553 |        0 | ehwmnuwmztrujxhtb65kpgq4sy | O    | Town Square | town-square                                            |        |         | 1474786510566 |             0 | 1474786510565 |                            |
| wi87emmg6786inbhgxzbnfgj3r | 1474786711899 | 1474786711899 |        0 | chjgwwy9qbbsdcz4ry4a6fbsoy | P    | XXX         | XXX                                                    |        |         | 1474786723310 |             1 | 1474786719125 | abkob7omkjynfqfsc8shkbwkfo |
+----------------------------+---------------+---------------+----------+----------------------------+------+-------------+--------------------------------------------------------+--------+---------+---------------+---------------+---------------+----------------------------+

mysql> select * from Posts;
+----------------------------+---------------+---------------+----------+----------------------------+----------------------------+----------------------------+----------------------------+------------+---------------------------------------+-------------------+-------+----------+-----------+
| Id                         | CreateAt      | UpdateAt      | DeleteAt | UserId                     | ChannelId                  | RootId                     | ParentId                   | OriginalId | Message                               | Type              | Props | Hashtags | Filenames |
+----------------------------+---------------+---------------+----------+----------------------------+----------------------------+----------------------------+----------------------------+------------+---------------------------------------+-------------------+-------+----------+-----------+
| 1mmp5u8t8fgffkkysrgcddj9oh | 1474786565691 | 1474786565691 |        0 | u7s1oanujifg8ywuirnmhk8r3a | 1muwbod1hjrmzcr6iukhzqk4dc |                            |                            |            | -                                     |                   | {}    |          | []        |
| 37wm6h78j7fh8dtq6p66uohjyy | 1474786329527 | 1474786329527 |        0 | abkob7omkjynfqfsc8shkbwkfo | 1h3rs83b3bnwzn3xb1ow49rssy | 811nz3334pbg78338ebbjfd8yy | 811nz3334pbg78338ebbjfd8yy |            | -                                     |                   | {}    |          | []        |
| 5ed5ghxp33gbik9denmqg17w6y | 1474786510566 | 1474786510566 |        0 | u7s1oanujifg8ywuirnmhk8r3a | g46cuaw3u781bjfyowudtfso7y |                            |                            |            | hithere has joined the channel.       | system_join_leave | {}    |          | []        |
| 811nz3334pbg78338ebbjfd8yy | 1474786320981 | 1474786329530 |        0 | abkob7omkjynfqfsc8shkbwkfo | 1h3rs83b3bnwzn3xb1ow49rssy |                            |                            |            | Hello                                 |                   | {}    |          | []        |
| anr5wpaoa3yuibqgmruiw98a5o | 1474786555809 | 1474786555809 |        0 | u7s1oanujifg8ywuirnmhk8r3a | 1h3rs83b3bnwzn3xb1ow49rssy |                            |                            |            | hithere has joined the channel.       | system_join_leave | {}    |          | []        |
| ct7wcfon87rnxkna313esudzhw | 1474786719128 | 1474786719128 |        0 | abkob7omkjynfqfsc8shkbwkfo | wi87emmg6786inbhgxzbnfgj3r |                            |                            |            | hithere added to the channel by hello | system_add_remove | {}    |          | []        |
| dmt68m5b3fd9zr16g9ci4mpdqc | 1474786777954 | 1474786777954 |        0 | abkob7omkjynfqfsc8shkbwkfo | f3j13bk4fi8qb8hh4mzgfbmmjc |                            |                            |            | -                                     |                   | {}    |          | []        |
| kss8nustk3nqjnnf9dsrw914my | 1474786311271 | 1474786311271 |        0 | abkob7omkjynfqfsc8shkbwkfo | 1muwbod1hjrmzcr6iukhzqk4dc |                            |                            |            | Hello!                                |                   | {}    |          | []        |
| maa1ttmi9pri8xwb9jekyci3hw | 1474786723309 | 1474786723309 |        0 | abkob7omkjynfqfsc8shkbwkfo | wi87emmg6786inbhgxzbnfgj3r |                            |                            |            | WTF                                   |                   | {}    |          | []        |
| soizm5ey3ig3bqkkygxjngodya | 1474786289489 | 1474786289489 |        0 | abkob7omkjynfqfsc8shkbwkfo | 1h3rs83b3bnwzn3xb1ow49rssy |                            |                            |            | hello has joined the channel.         | system_join_leave | {}    |          | []        |
| yj8aayxz4ir38m9r6b97cxbgco | 1474786524397 | 1474786524397 |        0 | u7s1oanujifg8ywuirnmhk8r3a | 3pocxn4za7rd58q6rzcycgerna |                            |                            |            | Jebus                                 |                   | {}    |          | []        |
+----------------------------+---------------+---------------+----------+----------------------------+----------------------------+----------------------------+----------------------------+------------+---------------------------------------+-------------------+-------+----------+-----------+
11 rows in set (0.00 sec)

Notes

  • There are 3 types of channels: open, private groups, direct messages
  • Direct messages have no team members. The channel name contains the members, e.g., “user_1__user_2”.
  • Search uses database’s full-text search by default.
  • Reads are performed on a read-only slave for high-availability: s.GetReplica().Select(&posts, “SELECT * FROM Posts”)
  • Writes are performed on the master: s.GetMaster().Exec(“Update Channels SET DeleteAt
  • Unread count is calculated using SQL and the cached values (channel.TotalMsgCount - channel_member.MsgCount):
-- unread count for user
SELECT
  SUM(CASE
    WHEN c.Type = 'D' THEN (c.TotalMsgCount - cm.MsgCount)
    ELSE cm.MentionCount
  END)
FROM Channels c
INNER JOIN ChannelMembers cm
  ON cm.ChannelId = c.Id
  AND cm.UserId = :UserId
  
-- unread count for user and channel
SELECT
  SUM(CASE
    WHEN c.Type = 'D' THEN (c.TotalMsgCount - cm.MsgCount)
    ELSE cm.MentionCount
  END)
FROM Channels c
INNER JOIN ChannelMembers cm
  ON c.Id = :ChannelId
  AND cm.ChannelId = :ChannelId
  AND cm.UserId = :UserId
  • Unread count is updated, e.g., when a channel is viewed. This involves storing the “last viewed” timestamp:
-- set last viewed at
UPDATE ChannelMembers
SET MentionCount = 0,
    MsgCount = Channels.TotalMsgCount - (SELECT
      COUNT(*)
    FROM Posts
    WHERE ChannelId = :ChannelId
    AND CreateAt > :NewLastViewedAt),
    LastViewedAt = :NewLastViewedAt
FROM Channels
WHERE Channels.Id = ChannelMembers.ChannelId
AND UserId = :UserId
AND ChannelId = :ChannelId

-- update last viewed at (new post, edit post, mark channel unread, etc)
UPDATE ChannelMembers
SET MentionCount = 0,
    MsgCount = Channels.TotalMsgCount,
    LastViewedAt = Channels.LastPostAt,
    LastUpdateAt = Channels.LastPostAt
FROM Channels
WHERE Channels.Id = ChannelMembers.ChannelId
AND UserId = :UserId
AND ChannelId = :ChannelId

Debugging tools

Tagged debugging, dstat, ngrep, perf, strace, tcpdump, wireshark, netcat, netstat, dtrace, dtruss  Languages bash

IO and system calls

  • dstat

Monitor network and disk IO:

dstat -t
  • dtrace / dtruss (OSX)

To get the list of available system calls use:

sudo dtrace -ln 'syscall:::entry'

Find which files a program is opening (same as strace -f -p $PID -e open):

sudo dtruss -t open_nocancel -p $PID

Also see ls /usr/bin/.d*

  • strace (Linux)

Monitor system calls made by an app:

strace ruby app.rb

Writes all system calls made by SSH, and subprocesses (-f), to a file named ssh.txt:

strace -f -o ssh.txt ssh jebus.com

Spy on all ‘open’ system calls made by a process:

strace -f -p $PID -e open

Use these commands to see a list of all available system calls (Linux only):

man syscalls
  • opensnoop

Monitor what files are being opened:

opensnoop -p $PID
strace -e open -p $PID

Networking

  • netcat

Pipe/copy data over a network:

cat request.txt | nc metafilter.com 80
  • netstat

Find which programs are listening to which port:

sudo netstat -tunapl
lsof -i -P # OSX
  • ngrep

Listen to traffic containing the string “localhost” on any network interface:

sudo ngrep -d any localhost
  • tcpdump

Listen to traffic containing the string “localhost” on any network interface:

sudo tcpdump port 80 -w http.pcap

Writes a pcap file that can be analyzed with Wireshark.

  • Wireshark

Analyze pcap files from ngrep, tcpdump, etc:

wireshark http.pcap

CPU (Linux)

  • perf

Run perf, a sampling profiler, to see where your application is spending its time:

sudo perf record ruby app.rb

Find out what the program using the most CPU time is doing:

sudo perf top

Find out if an app is using the L1 cache which is ~200 times faster than RAM:

sudo perf stat -e L1-dcache-load-misses my_golang_app

References

How to make Vim yank to the OSX clipboard

Tagged clipboard, osx, vim, yank  Languages vim, bash

Add this to .vimrc:

set clipboard=unnamed

If you use Tmux you probably need this: https://github.com/ChrisJohnsen/tmux-MacOSX-pasteboard

$ brew install reattach-to-user-namespace

Note, do NOT use something like this because it doesn’t work perfectly:

function! ClipboardYank()
  call system('pbcopy', @@)
endfunction
function! ClipboardPaste()
  let @@ = system('pbpaste')
endfunction

vnoremap <silent> y y:call ClipboardYank()<cr>
vnoremap <silent> d d:call ClipboardYank()<cr>
nnoremap <silent> p :call ClipboardPaste()<cr>p

How to get rid of Gulp, Grunt, and <your favorite JS build tool>

Tagged babel, browserify, build, grunt, gulp, npm  Languages bash

I would happily pay 50-100€ for a build tool that would help me get rid of Gulp, Grunt, etc, and save me from hours of unproductive fighting with unreliable tools.

The following snippet is not quite a full solution for getting rid of Gulp/Grunt.

Installation

npm install -g node-sass browserify babelify uglify-js

Usage

# Compile JS modules into one file
browserify js/app.js -o dist/js/app.js -t [ babelify --presets [ es2015 ] ]
# Compile CSS/SASS modules into one file
node-sass css/app.scss --output-style compressed --include-path node_modules/bootstrap-sass/assets/stylesheets dist/css/app.css
# Minify JS
uglifyjs --compress --mangle -o dist/js/app.js -- dist/js/app.js

Note:

See How to Use npm as a Build Tool.