sync_postboxes

#!/usr/bin/perl
use Modern::Perl;
use List::MoreUtils qw(uniq);
use Tie::RefHash;
use Getopt::Long;
use Date::Manip;
use lib '.';
use Postboxes;
use OSM;

# Get and validate command-line options.
our ($opt_cautious, $opt_timestamp) = (0, 1);
(GetOptions('cautious', 'timestamp!') && @ARGV == 1)
    or die <<END
usage: $0 [--cautious] [--no-timestamp] postcode-prefix
e.g. $0 E10
END
    ;
my $postcode_prefix = shift;

# Maximum distance for matching two postbox positions.
my $MAX_DISTANCE = 100;

# Fetch the data from both sources.
my (@dracos_data, @osm_data);
{
    my ($dr, $or)
	= Postboxes::get_two_data_sets $postcode_prefix, $MAX_DISTANCE;
    @dracos_data = @$dr;
    @osm_data = @$or;
}

# In --cautious mode we do not use the Post Office's address info.
if ($opt_cautious) {
    foreach (@dracos_data) {
	delete $_->{$_} foreach qw(postal_code addr:housenumber addr:street);
    }
}

# Given one data point, find the closest to it in a list of others,
# subject to $MAX_DISTANCE.
#
sub find_closest {
    my ($box, @others) = @_;
    my $lat = $box->{lat};
    my $lon = $box->{lon};
    my ($closest, $closest_distance);
    foreach my $n (@others) {
	my ($nlat, $nlon) = ($n->{lat}, $n->{lon});
	my $distance = OSM::distance $lat, $lon, $nlat, $nlon;
	next if $distance > $MAX_DISTANCE;
	if (not defined $closest_distance or $distance < $closest_distance) {
	    $closest = $n;
	    $closest_distance = $distance;
	}
    }
    return $closest;
}

# Find unambiguous is-closest-to matches.
my %closest_osm_to_dracos;
tie %closest_osm_to_dracos, 'Tie::RefHash' or die;
foreach (@dracos_data) {
    my $closest = find_closest $_, @osm_data;
    $closest_osm_to_dracos{$_} = $closest if $closest;
}
my %closest_dracos_to_osm;
tie %closest_dracos_to_osm, 'Tie::RefHash' or die;
foreach (@osm_data) {
    my $closest = find_closest $_, @dracos_data;
    $closest_dracos_to_osm{$_} = $closest if $closest;
}
my @unambiguous;
foreach my $d (keys %closest_osm_to_dracos) {
    my $o = $closest_osm_to_dracos{$d};
    my $got = $closest_dracos_to_osm{$o};
    if (defined $got and $got eq $d) {
	push @unambiguous, [ $d, $o ];
    }
}

my @attr_changes;
use constant EPSILON => 1;
foreach (@unambiguous) {
    my %dracos = %{$_->[0]};
    my %osm = %{$_->[1]};
    my %osm_new = %osm;
    my $changed = 0;
    my $distance = OSM::distance delete($dracos{lat}), delete($dracos{lon}),
                                 delete($osm{lat}), delete($osm{lon});
    delete($osm{amenity}) eq 'post_box' or die;
    my $id = delete $osm{id};
    my $version = delete $osm{version};

    # Check postal code.
    my $dracos_postal_code = delete $dracos{postal_code};
    my $osm_postal_code = delete $osm{postal_code};
    if (defined $dracos_postal_code and defined $osm_postal_code) {
	if ($dracos_postal_code eq $osm_postal_code) {
	    # OK.
	}
	elsif ($osm_postal_code =~ /\A${dracos_postal_code}\d+\z/) {
	    # The OSM one is more detailed.  Do not warn.
	}
	elsif ($dracos_postal_code =~ /\A$osm_postal_code/) {
	    $osm_new{postal_code} = $dracos_postal_code;
	    $changed = 1;
	}
	else {
	    warn qq{$id $version: postal_code $osm_postal_code is inconsistent with Dracos $dracos_postal_code\n};
	}
    }

    # Check 'ref' - sometimes on the map two are given.
    if (defined $dracos{ref} and defined $osm{ref} and $osm{ref} =~ tr/;//) {
	my $d = delete $dracos{ref};
	my @o = split /;\s*/, delete $osm{ref};
	if (@o ~~ $d) {
	    # OK, this ref is there, plus some others.
	}
	else {
	    warn qq{$id $version: refs (@o) are inconsistent with Dracos $d\n};
	}
    }

    foreach (sort keys %dracos) {
        my $d = $dracos{$_};
        my $o = $osm{$_};
        if (not defined $o) {
	    $osm_new{$_} = $d;
	    $changed = 1;
	}
	elsif ($d ne $o) {
	    warn qq{$id $version: discrepancy: Dracos $_="$d" vs OSM $_="$o"\n};
	}
    }

    if ($changed) {
	$osm_new{source} //= 'dracos';
	push @attr_changes, \%osm_new;
    }
}

my %has_unambiguous;
foreach (@unambiguous) {
    foreach (@$_) {
	$has_unambiguous{$_}++ && die;
    }
}

my @new_nodes;
my %suggested;
foreach my $d (@dracos_data) {
    next if $has_unambiguous{$d};
    my $lat = $d->{lat};
    my $lon = $d->{lon};
    my $loc1 = $d->{loc1};
    my $ref = $d->{ref};
    my $postal_code = $d->{postal_code};
    my $operator = $d->{operator};

    # Find nearby unmatched OSM nodes.
    my @osm_with_distance
	= map { [ $_, OSM::distance $lat, $lon, $_->{lat}, $_->{lon} ] }
    @osm_data;
    @osm_with_distance = grep { $_->[1] <= $MAX_DISTANCE } @osm_with_distance;
    @osm_with_distance
	= reverse sort { $a->[1] <=> $b->[1] } @osm_with_distance;

    if (@osm_with_distance) {
	my $msg = sprintf 'to add at (%.6f, %.6f), ref=%s', $lat, $lon, $ref;
	 warn "$msg\n";
	 warn "...but has nearby OSM postboxes:\n";
	foreach (@osm_with_distance) {
	    my ($n, $distance) = @$_;
	    $distance = int $distance;
	    print STDERR "  $n->{id} $n->{version}: $distance metres away ($n->{lat}, $n->{lon})\n";
            print STDERR '   (already matched)' if $has_unambiguous{$n};
            print STDERR "\n";
	    $suggested{$n} = 1;
	}
    }
    else {
	# Nothing nearby, we can safely add the new node.
	my $node_id = -(scalar(@new_nodes) + 1);
	my %h = (
	    id => $node_id, lat => $lat, lon => $lon,
	    amenity => 'post_box', ref => $ref, 
	    operator => $operator,
	    source => 'dracos', 'dracos:verified' => 'no',
	    );
	foreach ('addr:housenumber', 'addr:street', 'postal_code') {
	    $h{$_} = $d->{$_} if exists $d->{$_};
	}
	push @new_nodes, \%h;
    }
}

# Don't warn about every OSM box in that rectangle, just those that
# have the ref asked for.  (TODO find a convex hull or something.)
#
foreach (@osm_data) {
    next if $has_unambiguous{$_};
    next if $suggested{$_};
    my $ref = $_->{ref};
    next if not defined $ref;
    next unless $ref =~ /\A$postcode_prefix /o;
    warn "$_->{id}: not in Dracos data\n";
}

my $timestamp;
$timestamp = UnixDate 'now', '%O' if $opt_timestamp; # FIXME requires UTC time
my $changeset = 999;
sub write_node {
    my %h = %{shift()};

    # First get out the standard attrs.
    my %a;
    $a{$_} = delete $h{$_} foreach qw(id lat lon version);
    $a{version} //= 0;
    $a{timestamp} = $timestamp if $opt_timestamp;
    $a{changeset} = $changeset;
    my $attr_str
	= join ' ', map { $_ . '="' . delete($a{$_}) . '"' } sort keys %a;
    say "    <node $attr_str>";
    foreach (sort keys %h) {
	say qq{      <tag k="$_" v="$h{$_}" />};
    }
    say '    </node>';
}

my $gunk = 'version="0.3" generator="sync_postboxes"';
exit if not @attr_changes and not @new_nodes;
say "<osmChange $gunk>";
foreach (sort { $a->{id} <=> $b->{id} } @attr_changes) {
    say "  <modify $gunk>";
    write_node $_;
    say '  </modify>';
}
foreach (reverse sort { $a->{id} <=> $b->{id} } @new_nodes) {
    say "  <create $gunk>";
    write_node $_;
    say '  </create>';
}
say '</osmChange>';