use strict; use warnings; use Benchmark qw(cmpthese); use Sereal; use JSON::XS (); use JSON (); use Google::ProtocolBuffers::Dynamic; use Getopt::Long; { my $d = Google::ProtocolBuffers::Dynamic->new('scripts/profile'); $d->load_file("map.proto"); $d->load_file("person.proto"); $d->load_file("transform.proto"); $d->map({ package => 'profile', prefix => 'GPD::Profile', options => { decode_blessed => 0 }, }); GPD::Profile::Any->set_decoder_options({ fieldtable => 1, transform => $Google::ProtocolBuffers::Dynamic::Fieldtable::profile_decoder_transform, }); } my $sereal_encoder = Sereal::Encoder->new; my $sereal_decoder = Sereal::Decoder->new; my $json_decoder = JSON::XS->new; exit main(); sub main { my %suites = ( decode_maps => \&profile_decode_maps, decode_objects => \&profile_decode_objects, decode_transform => \&profile_decode_transform, encode_objects => \&profile_encode_objects, encode_maps => \&profile_encode_maps, ); my %callgrind_benchmarks = ( bbpb => 'protobuf_bbpb', upb => 'protobuf_upb', json => 'json', sereal => 'sereal', ); my $error; my $arg_ok = GetOptions( 'suite=s' => \(my $suite), 'callgrind=s' => \(my $callgrind_implementation), 'help' => \(my $help), ); if ($callgrind_implementation && !exists $callgrind_benchmarks{$callgrind_implementation}) { $error = "Unknown value '$callgrind_implementation' for implementation"; } if ($suite && !exists $suites{$suite}) { $error = "Unknown value '$suite' for suite"; } if ($callgrind_implementation && !$suite) { $error = "Can't profile with callgrind without selecting a suite"; } if (!$arg_ok || $help || $error) { help(!$arg_ok || !!$error, $error, \%suites); } my $with_callgrind = 0; my $repeat_count = $callgrind_implementation ? 2000 : -3; my $which_benchmarks = $callgrind_implementation ? [$callgrind_benchmarks{$callgrind_implementation}] : [qw(protobuf_bbpb protobuf_upb json sereal)]; setup(); for my $run_suite ($suite ? $suite : sort keys %suites) { print "\nRunning $run_suite\n\n"; start_callgrind() if $callgrind_implementation; $suites{$run_suite}->($repeat_count, $which_benchmarks); stop_callgrind() if $callgrind_implementation; } } sub help { my ($is_error, $error_msg, $suites) = @_; my $suites_list = join '', map " $_\n", keys %$suites; print $error_msg, "\n\n" if $error_msg; print <] [--suite=] --help This message --callgrind=[bbpb|upb|json|sereal] Only run the given implementation, enable callgrind while running the benchmark (the script needs to be run by using valgrind --tool=callgrind --instr-atstart=no ...) --suite= Only run the given banchmark suite, which can be one of: $suites_list EOT exit $is_error; } my %benchmarks; sub random_chars { my ($min, $max) = @_; my $num = rand($max - $min) + $min; return join '', map chr(rand(26) + 97), 1 .. $num; } sub setup { setup_encode_objects(); setup_decode_objects(); setup_encode_maps(); setup_decode_maps(); setup_decode_transform(); } sub setup_decode_maps { my $maps = $benchmarks{encode}{maps}; $benchmarks{decode}{maps}{protobuf} = GPD::Profile::Maps->encode($maps->{protobuf}); $benchmarks{decode}{maps}{sereal} = $sereal_encoder->encode($maps->{plain}); $benchmarks{decode}{maps}{json} = $json_decoder->encode($maps->{plain}); } sub profile_decode_maps { my ($repeat_count, $which_benchmarks) = @_; my $data = $benchmarks{decode}{maps}; cmpthese($repeat_count, filter_benchmarks($which_benchmarks, { protobuf_upb => sub { GPD::Profile::Maps->decode_upb($data->{protobuf}) }, protobuf_bbpb => sub { GPD::Profile::Maps->decode_bbpb($data->{protobuf}) }, sereal => sub { $sereal_decoder->decode($data->{sereal}) }, json => sub { $json_decoder->decode($data->{json}) }, })); } sub setup_decode_objects { my $persons = $benchmarks{encode}{objects}; $benchmarks{decode}{objects}{protobuf} = GPD::Profile::PersonArray->encode($persons); $benchmarks{decode}{objects}{sereal} = $sereal_encoder->encode($persons); $benchmarks{decode}{objects}{json} = $json_decoder->encode($persons); } sub profile_decode_objects { my ($repeat_count, $which_benchmarks) = @_; my $data = $benchmarks{decode}{objects}; cmpthese($repeat_count, filter_benchmarks($which_benchmarks, { protobuf_upb => sub { GPD::Profile::PersonArray->decode_upb($data->{protobuf}) }, protobuf_bbpb => sub { GPD::Profile::PersonArray->decode_bbpb($data->{protobuf}) }, sereal => sub { $sereal_decoder->decode($data->{sereal}) }, json => sub { $json_decoder->decode($data->{json}) }, })); } sub setup_decode_transform { my $make_random_any; $make_random_any = sub { my ($level) = @_; my $which = rand; if ($which < .10 / $level) { my ($map, $map_pb) = ({}, {}); for my $i (0 .. rand(10)) { my $key = random_chars 2, 4; ($map->{$key}, $map_pb->{$key}) = $make_random_any->($level + 1); } return ($map, { map_value => $map_pb }); } elsif ($which < .20 / $level) { my ($array, $array_pb) = ([], []); for my $i (0 .. rand(10)) { ($array->[$i], $array_pb->[$i]) = $make_random_any->($level + 1); } return ($array, { array_value => $array_pb }); } elsif ($which < .50) { my $value = 1 + int(rand 100); return ($value, { int64_value => $value }); } else { my $string = random_chars(6, 12); return ($string, { string_value => $string }); } }; my ($values, $values_pb); for my $i (1 .. 100) { ($values->{$i}, $values_pb->{$i}) = $make_random_any->(1); } $benchmarks{decode}{transform}{protobuf} = GPD::Profile::Values->encode({ values => $values_pb }); $benchmarks{decode}{transform}{sereal} = $sereal_encoder->encode({ values => $values }); $benchmarks{decode}{transform}{json} = $json_decoder->encode({ values => $values }); } sub profile_decode_transform { my ($repeat_count, $which_benchmarks) = @_; my $data = $benchmarks{decode}{transform}; cmpthese($repeat_count, filter_benchmarks($which_benchmarks, { protobuf_upb => sub { GPD::Profile::Values->decode_upb($data->{protobuf}) }, protobuf_bbpb => sub { GPD::Profile::Values->decode_bbpb($data->{protobuf}) }, sereal => sub { $sereal_decoder->decode($data->{sereal}) }, json => sub { $json_decoder->decode($data->{json}) }, })); } sub setup_encode_objects { my $make_random_person = sub { my ($id) = @_; return { id => $id, name => random_chars(3, 16) . ' ' . random_chars(6, 12), email => random_chars(7, 12) . '@test.com', }; }; my $persons = { persons => [map $make_random_person->($_), 1 .. 100 ], }; $benchmarks{encode}{objects} = $persons; } sub profile_encode_objects { my ($repeat_count, $which_benchmarks) = @_; my $data = $benchmarks{encode}{objects}; cmpthese($repeat_count, filter_benchmarks($which_benchmarks, { protobuf_upb => sub { GPD::Profile::PersonArray->encode($data) }, sereal => sub { $sereal_encoder->encode($data) }, json => sub { $json_decoder->encode($data) }, })); } sub setup_encode_maps { my $make_string_int32_map = sub { return +{ map +(random_chars(5, 15) => int(rand(1000) + 2)), (1 .. 30) }; }; my $make_string_string_map = sub { return +{ map +(random_chars(5, 15) => random_chars(5, 15)), (1 .. 30) }; }; my $maps = { string_int32_maps => [ map +($make_string_int32_map->()), (1 .. 10), ], string_string_maps => [ map +($make_string_string_map->()), (1 .. 10), ], }; my $proto_maps = { string_int32_maps => [ map +{ string_int32_map => $_ }, @{$maps->{string_int32_maps}} ], string_string_maps => [ map +{ string_string_map => $_ }, @{$maps->{string_string_maps}} ], }; $benchmarks{encode}{maps}{protobuf} = $proto_maps; $benchmarks{encode}{maps}{plain} = $maps; } sub profile_encode_maps { my ($repeat_count, $which_benchmarks) = @_; my $data = $benchmarks{encode}{maps}; cmpthese($repeat_count, filter_benchmarks($which_benchmarks, { protobuf_upb => sub { GPD::Profile::Maps->encode($data->{protobuf}) }, sereal => sub { $sereal_encoder->encode($data->{plain}) }, json => sub { $json_decoder->encode($data->{plain}) }, })); } sub start_callgrind { system "callgrind_control --instr=on $$"; } sub stop_callgrind { system "callgrind_control --instr=off $$"; } sub filter_benchmarks { my ($which, $benchmarks) = @_; my %which; @which{@$which} = (); return { map +(exists $which{$_} ? ($_ => $benchmarks->{$_}) : ()), keys %$benchmarks }; }