Use CSV as it's much quicker and easier than scraping HTML :-)
This commit is contained in:
		
							
								
								
									
										51
									
								
								csv.pl
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										51
									
								
								csv.pl
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,51 @@ | ||||
| #!/usr/bin/env perl -w | ||||
| # | ||||
| $|=1 if $ENV{DEBUG}; | ||||
| use strict; | ||||
| use diagnostics; | ||||
| use DBI; | ||||
| use Text::CSV; | ||||
| use Date::Manip; | ||||
|  | ||||
| use LWP::UserAgent; | ||||
| my $ua = LWP::UserAgent->new; | ||||
| $ua->agent("Munin[river_levels_]"); | ||||
|  | ||||
| my @stations = qw( | ||||
| 1129 1135 1133 1143 1132 1137 1134 1145 1139 1131 1130 1144 9274 | ||||
| ); | ||||
|  | ||||
| my $base = "https://check-for-flooding.service.gov.uk/station-csv/"; | ||||
|  | ||||
| my $db = DBI->connect("dbi:SQLite:dbname=rivers.db") or die $DBI::errstr; | ||||
|  | ||||
| my $recordlevel    = $db->prepare("REPLACE INTO levels (stationid, timestamp, level) VALUES (?,?,?)") or die $db->errstr; | ||||
| my $getlastrecord  = $db->prepare("SELECT MAX(timestamp) FROM levels WHERE stationid=?")              or die $db->errstr; | ||||
|  | ||||
| sub getlatest($) { | ||||
|         my ($id) = @_; | ||||
|         $getlastrecord->execute($id); | ||||
|         my ($lastrecord) = $getlastrecord->fetchrow(); | ||||
| 	return $lastrecord ? $lastrecord : "0"; | ||||
| } | ||||
|  | ||||
|  | ||||
| foreach my $station (@stations) { | ||||
|         print STDERR "[$station] Requesting $base$station\n" if $ENV{R_DEBUG}; | ||||
|         my $req = HTTP::Request->new(GET => $base . $station); | ||||
|         my $res = $ua->request($req); | ||||
| 	my $content = $res->content; | ||||
| 	my $last = getlatest($station); | ||||
| 	print "[$station] Last record was $last\n" if $ENV{R_DEBUG}; | ||||
| 	foreach my $line (sort split(/^/m, $content)) { | ||||
| 		next if $line =~ /Timestamp (UTC)/; | ||||
| 		chomp($line); | ||||
| 		my ($ts, $level) = split(/,/, $line); | ||||
| 		if($ts le $last) { | ||||
| 			print "[$station] Skipping $level at $ts\n" if $ENV{R_DEBUG}; | ||||
| 		} else { | ||||
| 			print "[$station] Recording $level at $ts\n" if $ENV{R_DEBUG}; | ||||
| 			$recordlevel->execute($station, $ts, $level); | ||||
| 		} | ||||
| 	} | ||||
| } | ||||
		Reference in New Issue
	
	Block a user