log snippets

Perl script that can be used to calculate min, max, mean, mode, median and standard deviation for a set of log records

Tagged perl, min, max, csv, mean, log, parser  Languages perl

The best thing about this script is that it's easy to customize, right now it's optimized for comma delimited data.

use strict;
use warnings;

# Import stdev, average, mean and other statistical functions
# A copy of http://search.cpan.org/~brianl/Statistics-Lite-3.2/Lite.pm

my %page_runtimes;
my $delimitor = ';';
my @columns = ("page", "samples", "min", "max", "mean", "mode", "median", "stddev\n");
my $line;
my $first_timestamp, my $last_timestamp;

# ==========================================
# Parse log file
# ==========================================

# Don't use foreach as it reads the whole file into memory: foreach $line (<>) { 
while ($line=<>) {
  # remove the newline from $line, otherwise the report will be corrupted.

  my @columns               = split(';', $line);
  my $timestamp             = $columns[0];
  my $page_name             = $columns[1];
  my $page_runtime          = $columns[2];

    $first_timestamp = $timestamp;

  # print what we find
    print "Found page '$page_name'\n";
  # add page runtimes to one hash
  push(@{$page_runtimes{$page_name}}, $page_runtime);
  $last_timestamp = $timestamp;

# ==========================================
# Calculate and print page statistics
# ==========================================
open(PAGE_REPORT, ">report.csv") or die("Could not open report.csv.");

print PAGE_REPORT "First sample\n".$first_timestamp."\nLast sample\n".$last_timestamp."\n\n";
print PAGE_REPORT join($delimitor, @columns);

for my $page_name (keys %page_runtimes )
  my @runtimes = @{$page_runtimes{$page_name}};
  my $samples = @runtimes;
  my $min     = min(@runtimes);
  my $max     = max(@runtimes);
  my $mean    = mean(@runtimes);
  my $mode    = mode(@runtimes);
  my $median  = median(@runtimes);
  my $stddev  = stddev(@runtimes);
  my @data = ($page_name, $samples, $min, $max, $mean, $mode, $median, $stddev);
  my $line = join($delimitor, @data);
  # Use comma instead of decimal
  $line =~ s/\./\,/g;
  print PAGE_REPORT "$line\n";

To use it simply pipe some data into it like this:

grep "2008-31-12" silly-data.log | perl analyze.pl

Exploring log wrappers for Go

Tagged wrapper, go, log, logging, debug, info  Languages go

Version 1

This is a simple log wrapper for go’s built in log package. Put the following code in a file named log.go:

package main

import (
    golog "log"

// Use ioutil.Discard instead of f to send log to '/dev/null'
type Logger struct {
    Info     *golog.Logger
    Error    *golog.Logger
    Debug    *golog.Logger
    FatalLog *golog.Logger
func (logger Logger) Fatal(format string, args ...interface{}) {
    logger.FatalLog.Printf(format, args...)
    golog.Fatalf(format, args...)

var logFile *os.File
var log Logger

func init() {
    fileName := os.Getenv("LOG")
    if len(fileName) > 0 {
        var err error
        logFile, err = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
        if err != nil {
            golog.Fatalf("error opening file: %v", err)
    } else {
        logFile = os.Stdout
    log.Info = golog.New(logFile, "INFO  ", golog.LstdFlags)
    log.Error = golog.New(logFile, "ERROR ", golog.LstdFlags)
    log.Debug = golog.New(logFile, "DEBUG ", golog.LstdFlags)
    log.FatalLog = golog.New(logFile, "FATAL ", golog.LstdFlags)


log.Info.Printf("Notifications %v", notifications)
log.Debug.Printf("Notifications %v", notifications)

Version 2

To remove the call to Printf you could do this:

type Log struct {
    //Info     *golog.Logger
    Info     func(string, ...interface{})

func init() {
    log.Info = golog.New(logFile, "INFO  ", golog.LstdFlags).Printf

Now you can simply call log.Info:

log.Info("Notifications %v", notifications)
log.Debug("Notifications %v", notifications)

See https://golang.org/pkg/log/#Logger

Version 3

An even simpler log wrapper:

package main

import (

var debug bool

func init() {
    debug = os.Getenv("DEBUG") != ""

func Log(msg string, vars ...interface{}) {
    log.Printf(strings.Join([]string{"[INFO ]", msg}, " "), vars...)

func Debug(msg string, vars ...interface{}) {
    if debug {
        log.Printf(strings.Join([]string{"[DEBUG]", msg}, " "), vars...)

Analyze Nginx's access logs with goaccess

Tagged log, access, analytics, goaccess  Languages bash

Show all statistics

sudo zcat /var/log/nginx/*.access.log.* | goaccess --log-format=COMBINED

Show server errors

sudo zcat /var/log/nginx/*.access.log.* | awk '$9~/500/' | goaccess --log-format=COMBINED