diff options
Diffstat (limited to 'util')
| -rw-r--r-- | util/check.sh | 43 | ||||
| -rw-r--r-- | util/checkasm.sh | 46 | ||||
| -rw-r--r-- | util/notime.pl | 26 | ||||
| -rw-r--r-- | util/overlap.sh | 27 | ||||
| -rw-r--r-- | util/shortf.pl | 31 | ||||
| -rw-r--r-- | util/table.pl | 360 | ||||
| -rw-r--r-- | util/uncompr.pl | 32 |
7 files changed, 565 insertions, 0 deletions
diff --git a/util/check.sh b/util/check.sh new file mode 100644 index 0000000..6fbd141 --- /dev/null +++ b/util/check.sh @@ -0,0 +1,43 @@ +#! /bin/sh +set -e + +# +# usage: util/check.sh [directory] +# +# This script runs lzotest with all algorithms on a complete directory tree. +# It is not suitable for accurate timings. +# +# Copyright (C) 1996-2011 Markus Franz Xaver Johannes Oberhumer +# + +LZOTEST="lzotest" +test -x ./lzotest/lzotest && LZOTEST="./lzotest/lzotest" +test -x ./lzotest.exe && LZOTEST="./lzotest.exe" +test -x ./lzotest.out && LZOTEST="./lzotest.out" + +dir="${*-.}" + +TMPFILE="/tmp/lzo_$$.tmp" +rm -f $TMPFILE +(find $dir/ -type f -print > $TMPFILE) || true + +## methods=`$LZOTEST -m | sed -n 's/^ *-m\([0-9]*\).*/\1/p'` +## methods="9721 9722 9723 9724 9725 9726 9727 9728 9729" +methods="21 31 1 2 3 4 5 6 7 8 9 11 12 13 14 15 16 17 18 19 61 71 81" +methods="$methods 111 112 115" +methods="$methods 921 931 901 911" +methods="$methods 902 912 942 962 972 982 992" +##methods="71 972" +##methods="1101 1102 1103 1104 1105 1106 1107" + +LFLAGS="-q -T -n2 -S" +LFLAGS="-q -T -n2" + +for i in $methods; do + cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS +done + +rm -f $TMPFILE +echo "Done." +exit 0 + diff --git a/util/checkasm.sh b/util/checkasm.sh new file mode 100644 index 0000000..63ed2b8 --- /dev/null +++ b/util/checkasm.sh @@ -0,0 +1,46 @@ +#! /bin/sh +set -e + +# +# usage: util/checkasm.sh [directory] +# +# This script runs lzotest with all assembler decompressors +# on a complete directory tree. +# It is not suitable for accurate timings. +# +# Copyright (C) 1996-2011 Markus Franz Xaver Johannes Oberhumer +# + +LZOTEST="lzotest" +test -x ./lzotest/lzotest && LZOTEST="./lzotest/lzotest" +test -x ./lzotest.exe && LZOTEST="./lzotest.exe" +test -x ./lzotest.out && LZOTEST="./lzotest.out" +LFLAGS="-q" + +dir="${*-.}" + +TMPFILE="/tmp/lzo_$$.tmp" +rm -f $TMPFILE +(find $dir/ -type f -print > $TMPFILE) || true + +for i in 11; do + cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -A + cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -A -S +done + +for i in 61; do + cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -F + cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -F -S +done + +for i in 71 81; do + cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -A + cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -A -S + cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -F + cat $TMPFILE | $LZOTEST -m${i} -@ $LFLAGS -F -S +done + +rm -f $TMPFILE +echo "Done." +exit 0 + diff --git a/util/notime.pl b/util/notime.pl new file mode 100644 index 0000000..9d8a6d1 --- /dev/null +++ b/util/notime.pl @@ -0,0 +1,26 @@ +#! /usr/bin/perl +## +## vi:ts=4 +## +##---------------------------------------------------------------------------## +## +## Author: +## Markus F.X.J. Oberhumer <markus@oberhumer.com> +## +## Description: +## Remove timing values from a table created by table.pl +## +## Copyright (C) 1996-2011 Markus Franz Xaver Johannes Oberhumer +## +##---------------------------------------------------------------------------## + + +while (<>) { + if (substr($_,56) =~ /^\s+[\d\.]+\s+[\d\.]+\s+\|\s*\n$/) { + substr($_,56) = " 0.000 0.000 |\n"; + } + print; +} + +exit(0); + diff --git a/util/overlap.sh b/util/overlap.sh new file mode 100644 index 0000000..727c86f --- /dev/null +++ b/util/overlap.sh @@ -0,0 +1,27 @@ +#! /bin/sh +set -e + +# +# usage: util/overlap.sh [directory] +# +# This script runs the overlap example program on a complete directory tree. +# +# Copyright (C) 1996-2011 Markus Franz Xaver Johannes Oberhumer +# + +OVERLAP="overlap" +test -x ./examples/overlap && OVERLAP="./examples/overlap" +test -x ./overlap.exe && OVERLAP="./overlap.exe" + +dir="${*-.}" + +TMPFILE="/tmp/lzo_$$.tmp" +rm -f $TMPFILE +(find $dir/ -type f -print0 > $TMPFILE) || true + +cat $TMPFILE | xargs -0 -r $OVERLAP + +rm -f $TMPFILE +echo "Done." +exit 0 + diff --git a/util/shortf.pl b/util/shortf.pl new file mode 100644 index 0000000..380d3eb --- /dev/null +++ b/util/shortf.pl @@ -0,0 +1,31 @@ +#! /usr/bin/perl +## +## vi:ts=4 +## +##---------------------------------------------------------------------------## +## +## Author: +## Markus F.X.J. Oberhumer <markus@oberhumer.com> +## +## Description: +## Create short files for compression test +## +## Copyright (C) 1996-2011 Markus Franz Xaver Johannes Oberhumer +## +##---------------------------------------------------------------------------## + +$c = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; +$c = "\x00\x01\x02"; +$c = "\x00"; + +$x = $c x 1024; + +for $i (0 .. 50) { + $name = sprintf("f%04d.dat",$i); + open(OUTFILE,">$name") || die "$0: cannot open '$name': $!"; + binmode(OUTFILE); + print OUTFILE substr($x,0,$i); + close(OUTFILE); +} + +exit(0); diff --git a/util/table.pl b/util/table.pl new file mode 100644 index 0000000..8ccbad6 --- /dev/null +++ b/util/table.pl @@ -0,0 +1,360 @@ +#! /usr/bin/perl +## +## vi:ts=4:et +## +##---------------------------------------------------------------------------## +## +## Author: +## Markus F.X.J. Oberhumer <markus@oberhumer.com> +## +## Description: +## Convert the output of the LZO lzotest program into a nice table. +## +## Copyright (C) 1996-2011 Markus Franz Xaver Johannes Oberhumer +## +##---------------------------------------------------------------------------## + +$PROG = $0; +require 'ctime.pl'; + +# +# get options +# + +while ($_ = $ARGV[ $[ ], /^-/) { + shift(@ARGV); + /^--$/ && ($opt_last = 1, last); + + /^--sort=name/ && ($opt_sort_summary_by_name++, next); + /^--sort=ratio/ && ($opt_sort_summary_by_ratio++, next); + /^-s/ && ($opt_summary_only++, next); + /^-t/ && ($opt_clear_time++, next); +} + + +$alg = ''; +$sep = "+" . ("-" x 76) . "+\n"; + +$block_size = -1; + +$n = 0; +@algs = (); +%average = (); +%total = (); + +$lzo_version_string = ''; +$lzo_version_date = ''; + + +# /*********************************************************************** +# // +# ************************************************************************/ + +while (<>) { + + if (/(^|\s)(\d+)\s+block\-size/i) { + if ($block_size < 0) { + $block_size = $2; + &intro($block_size); + } elsif ($block_size != $2) { + die "$PROG: block-size: $block_size != $2\n"; + } + next; + } + + if (/^\s*LZO\s.*library\s+\(v\s*([\w\.\s]+)\s*\,\s*([^\)]+)\)/) { + $lzo_version_string = $1; + $lzo_version_date = $2; + next; + } + + if (/^\s*(\S+(\s+\[\S+\])?)\s*(\|.*\|)\s*$/i) { + if ($1 ne $alg) { + &footer($1); + &header($1); + } + $line = $3; + &stats(*line); + print "$line\n" if (!$opt_summary_only); + } +} +&footer($1); + +&summary(); + +exit(0); + + +# /*********************************************************************** +# // +# ************************************************************************/ + +sub stats { + local (*l) = @_; + local ($x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8); + + if ($l !~ /^\|\s*(.+?)\s+(\d+)\s+(\d+)\s+(\d+)\s+([\d\.]+\s+)?([\d\.]+\s+)?([\d\.]+)\s+([\d\.]+)\s*\|/) { + die $_; + } + + $n++; + + $x1 = $1; $x2 = $2; $x3 = $3; $x4 = $4; + $x5 = ($x2 > 0) ? $x4 * 100.0 / $x2 : 0.0; + $x6 = ($x2 > 0) ? $x4 * 8.0 / $x2 : 0.0; + $x7 = $7; $x8 = $8; + + # convert from kB/s to MB/s (for old versions of lzotest) + if ($x7 =~ /\.\d\d$/) { $x7 = $x7 / 1000.0; } + if ($x8 =~ /\.\d\d$/) { $x8 = $x8 / 1000.0; } + + if ($opt_clear_time) { + $x7 = $x8 = 0.0; + } + + $s[0] += $x2; + $s[1] += $x3; + $s[2] += $x4; + $s[3] += $x5; + $s[4] += $x6; + if ($x7 > 0) { + $s[5] += 1.0 / $x7; $sn[5] += 1; + } + if ($x8 > 0) { + $s[6] += 1.0/ $x8; $sn[6] += 1; + } + + $x1 =~ s/\s+$//; + $l = sprintf("| %-14s %10d %5d %9d %6.1f %5.2f %9.3f %9.3f |", + $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8); +} + + +# /*********************************************************************** +# // +# ************************************************************************/ + +sub header { + local ($t) = @_; + + $alg = $t; + + # reset stats + $n = 0; + @s = (0, 0, 0, 0.0, 0.0, 0.0, 0.0); + @sn = (0, 0, 0, 0, 0, 0, 0); + + return if $opt_summary_only; + + print "\n$alg\n\n"; + print $sep; +print <<EndOfString; +| File Name Length CxB ComLen Ratio% Bits Com MB/s Dec MB/s | +| --------- ------ --- ------ ----- ---- -------- -------- | +EndOfString +} + + +# /*********************************************************************** +# // +# ************************************************************************/ + +sub footer { + local ($t) = @_; + local ($shm5, $shm6); + + return unless $alg; + die if $n <= 0; + die if $s[0] <= 0; + + # harmonic mean + $shm5 = $s[5] > 0 ? $sn[5] / $s[5] : 0.0; + $shm6 = $s[6] > 0 ? $sn[6] / $s[6] : 0.0; + + push(@algs,$alg); + + $average{$alg} = + sprintf("| %-14s %10d %5d %9d %6.1f %5.2f %9.3f %9.3f |\n", + "Average", $s[0]/$n, $s[1]/$n, $s[2]/$n, + $s[3]/$n, $s[4]/$n, + $shm5, $shm6); + + $total{$alg} = + sprintf("| %-14s %10d %5d %9d %6.1f %5.2f %9.3f %9.3f |\n", + "Total", $s[0], $s[1], $s[2], + $s[2]/$s[0]*100, $s[2]/$s[0]*8, + $shm5, $shm6); + + return if $opt_summary_only; + + print $sep; + print $average{$alg}; + print $total{$alg}; + print $sep, "\n"; +} + + +# /*********************************************************************** +# // +# ************************************************************************/ + +$sort_mode = 0; + +sub cmp_by_ratio { + local ($aa, $bb); + + if ($sort_mode == 0) { + $aa = $average{$a}; + $bb = $average{$b}; + } elsif ($sort_mode == 1) { + $aa = $total{$a}; + $bb = $total{$b}; + } else { + die; + } + + ($aa =~ m%^\s*\|\s+\S+\s+\d+\s+\d+\s+\d+\s+(\S+)%) || die; + $aa = $1; + ($bb =~ m%^\s*\|\s+\S+\s+\d+\s+\d+\s+\d+\s+(\S+)%) || die; + $bb = $1; + + # $aa < $bb; + $aa cmp $bb; +} + + +# /*********************************************************************** +# // +# ************************************************************************/ + +sub summary { + local ($l); + local (@k); + + $sort_mode = 0; + if ($opt_sort_summary_by_name) { + @k = sort(@algs); + } elsif ($opt_sort_summary_by_ratio) { + @k = sort(cmp_by_ratio @algs); + } else { + @k = @algs; + } + + print "\n\n"; + print "Summary of average values\n\n"; + print $sep; +print <<EndOfString; +| Algorithm Length CxB ComLen Ratio% Bits Com MB/s Dec MB/s | +| --------- ------ --- ------ ----- ---- -------- -------- | +EndOfString + + for (@k) { + $l = $average{$_}; + $l =~ s/Average[\s]{7}/sprintf("%-14s",$_)/e; + print $l; + } + print $sep; + + + + $sort_mode = 1; + if ($opt_sort_summary_by_name) { + @k = sort(@algs); + } elsif ($opt_sort_summary_by_ratio) { + @k = sort(cmp_by_ratio @algs); + } else { + @k = @algs; + } + + print "\n\n"; + print "Summary of total values\n\n"; + print $sep; +print <<EndOfString; +| Algorithm Length CxB ComLen Ratio% Bits Com MB/s Dec MB/s | +| --------- ------ --- ------ ----- ---- -------- -------- | +EndOfString + + for (@k) { + $l = $total{$_}; + $l =~ s/Total[\s]{9}/sprintf("%-14s",$_)/e; + print $l; + } + print $sep; +} + + +# /*********************************************************************** +# // +# ************************************************************************/ + +sub intro { + local ($bs) = @_; + local ($v, $t, $x); + local ($u, $uname_m, $uname_s, $uname_r); + + $t = &ctime(time); chop($t); + $t = sprintf("%-55s |", $t); + + $v=''; + if ($lzo_version_string) { + $v = $lzo_version_string; + $v .= ', ' . $lzo_version_date if $lzo_version_date; + $v = sprintf("%-55s |", $v); + $v = sprintf("| LZO version : %s\n", $v); + } + + if ($bs % 1024 == 0) { + $x = sprintf("%d (= %d kB)", $bs, $bs / 1024); + } else { + $x = sprintf("%d (= %.3f kB)", $bs, $bs / 1024.0); + } + $x = sprintf("%-55s |", $x); + + $u=''; + if (1 == 1) { + $uname_s = `uname -s`; $uname_s =~ s/^\s+//; $uname_s =~ s/\s+$//; + $uname_r = `uname -r`; $uname_r =~ s/^\s+//; $uname_r =~ s/\s+$//; + $uname_m = `uname -m`; $uname_m =~ s/^\s+//; $uname_m =~ s/\s+$//; + if ($uname_s && $uname_m) { + $u = $uname_s; + $u .= ' ' . $uname_r if $uname_r; + $u .= ' ' . $uname_m; + $u = sprintf("%-55s |", $u); + $u = sprintf("| Operating system : %s\n", $u); + } + } + print <<EndOfString; + ++----------------------------------------------------------------------------+ +| DATA COMPRESSION TEST | +| ===================== | +| Time of run : $t +$v$u| Context length : $x ++----------------------------------------------------------------------------+ + + +Notes: +- CxB is the number of independent blocks a file was splitted +- MB/s is the speed measured in 1,000,000 uncompressed bytes per second +- all averages are calculated from the un-rounded values +- the average ratio & bits are calculated by the arithmetic mean +- the average speed is calculated by the harmonic mean + + +EndOfString +} + +__END__ + + +### insert something like this after 'Time of run': + +| Hardware : Intel Pentium 133, 64 MB RAM, 256 kB Cache | +| Operating system : MS-DOS 7.10, HIMEM.SYS 3.95, DOS/4GW 1.97 | +| Compiler : Watcom C32 10.5 | +| Compiler flags : -mf -5r -oneatx | +| Test suite : Calgary Corpus Suite | +| Files in suite : 14 | +| Timing accuracy : One part in 100 | + + diff --git a/util/uncompr.pl b/util/uncompr.pl new file mode 100644 index 0000000..df3e969 --- /dev/null +++ b/util/uncompr.pl @@ -0,0 +1,32 @@ +#! /usr/bin/perl +## +## vi:ts=4 +## +##---------------------------------------------------------------------------## +## +## Author: +## Markus F.X.J. Oberhumer <markus@oberhumer.com> +## +## Description: +## Create incompressible files for compression test +## +## Copyright (C) 1996-2011 Markus Franz Xaver Johannes Oberhumer +## +##---------------------------------------------------------------------------## + +$x = ' ' x 65536; +$i = 0; +while ($i < 65536) { + substr($x,$i,1) = pack('C',rand(256)); + $i++; +} + +for $i (1,2,4,8,16,32,64) { + $name = sprintf("u%04d.dat",$i); + open(OUTFILE,">$name") || die "$0: cannot open '$name': $!"; + binmode(OUTFILE); + print OUTFILE substr($x,0,$i*1024); + close(OUTFILE); +} + +exit(0); |