#!/usr/bin/perl

# Crawl post office finder web site
# Copyright (C) 2008  Edward Betts
# 
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

use strict;
use warnings;
use Data::Dump 'dump';
use WWW::Mechanize;
use File::Slurp;

my $mech = WWW::Mechanize->new();
$mech->add_header('Accept' => 'text/html');

my $url = 'http://www.postoffice.co.uk/portal/po/finder';
$url = 'http://www.postoffice.co.uk/portal/po/finder?catId=20700386';

my $searchType = "/rmg/finder/FindTheNearestFormHandler.value.searchCodeType";
my $postTown = "/rmg/finder/FindTheNearestFormHandler.value.postTown";

read_town($_) for 'A'..'Z';

sub read_town {
    my $town = shift;
    print "$url\n";
    try_get($url);
    $mech->success() or die;
    my $res = $mech->submit_form(
        form_name => 'finderForm',
        fields    => { $searchType => 'postTownSearch', $postTown => $town, },
        button    => '/rmg/finder/FindTheNearestFormHandler.search',
    );

    my $page_num = 1;

    my $next_link;
    do { 
        my @detail_urls;

        for (1..5) {
            @detail_urls = $mech->find_all_links( url_regex => qr/postofficeIndex/);
            @detail_urls and last;
            print "error, trying again\n";
            sleep 10;
            $mech->reload;
        }

        @detail_urls or die;

        $next_link = $mech->find_link( text_regex => qr/^next/i );

        for (@detail_urls) {
            my ($index) = $_->url =~ /postofficeIndex=(\d|10)$/ or die;
            try_get($_);
            my $filename = sprintf "pages/%s_%04d_%02d.html", $town, $page_num, $index;
            write_file($filename, $mech->content);
            print "$filename\n";
        }
        $page_num++;
        if ($next_link) {
            print "$next_link\n";
            try_get($next_link);
        }
    } while ($next_link);
}

sub try_get {
    my $url = shift;
    for (1..5) {
        $mech->get($url);
        $mech->success() and last;
        print "error, trying again\n";
        sleep 10;
    }
    $mech->success() or die "too many errors";
    $mech->content =~ m/An error has occured\./ and die;
}
