mafoo.org.uk/rivers
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

another EA data change

Browse Source
This commit is contained in:
foo 2016-03-25 07:07:50 +00:00
parent 6b68c4b418
commit 00d601fca5
6 changed files with 151 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
getlast3mo.sh Executable file
View File

@ -0,0 +1 @@
sqlite3 -init plot.sqliterc rivers.db "select timestamp, level from levels where stationid = $1 and timestamp > date('now', '-3 months')"

22
getrainfall.pl Normal file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env perl -w
# Reformat data from Metoffice for use in GNUPlot
# curl http://www.metoffice.gov.uk/hadobs/hadukp/data/daily/HadSEEP_daily_qc.txt | perl getrainfall.pl
use strict;
<>; <>; <>; # swallow first three lines
while(<>) {
chomp;
s/^\s*//;
my ($year, $month, @data) = split (/\s+/);
next unless $year >= 2014;
my $day = 1;
foreach(@data) {
next if $_ < 0;
printf("%04d-%02d-%02d %s\n", $year, $month, $day, $_);
$day++;
}
}

29
new.getstations.pl Executable file
View File

@ -0,0 +1,29 @@
#!/usr/bin/perl -w
$|=1;
use strict;
use diagnostics;
use DBI;
use LWP::UserAgent;
my $ua = LWP::UserAgent->new;
$ua->agent("Munin[river_levels_]");
my $base = "http://apps.environment-agency.gov.uk/river-and-sea-levels/";
my $req = HTTP::Request->new(GET => $base . $ARGV[0] . ".aspx");
my $res = $ua->request($req);
if ($res->is_success) {
my @lines = split("\n", $res->content);
foreach my $line (@lines) {
if( $line =~ m~href="\?stationId=(\d+)" alt="([^"]+)"~ ) {
my $stationId = $1;
my $name = $2;
print $stationId . " ";
}
}
} else {
die "failed!";
}

34
new.import.pl Executable file
View File

@ -0,0 +1,34 @@
#!/usr/bin/perl -w
$|=1;
use strict;
use diagnostics;
use DBI;
use POSIX;
my @stations = qw(
1129 1135 1133 1143 1132 1137 1134 1145 1139 1131 1130 1144
1086 1085 1073 1091 1071 1076 1069 1066 1067 1072 1074 1083 1064 1090 1089 1088 1087 1078 1080 1063
1030 1120 1121 1122 1125 1126 1127
6074 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6246 6247 6249 6250 6251 6253 6254 6255 6257 6259 6260 6262 6265 6267 6270 6271 6272 6273 6275 6278 6279 6280 6281
6245 6252 6256 6261
6009 6248 6268 6277 6269
6075 6076 6077 6078 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6263 6264 6266 6276 6258 6164 6274
);
my $db = DBI->connect("dbi:SQLite:dbname=rivers.db") or die $DBI::errstr;
my $in = $db->prepare("INSERT INTO levels (stationid, timestamp, level) VALUES (?,?,?)") or die $db->errstr;
while(<>) {
chomp;
my ($time, @data) = split(/\t/);
# my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime($time);
# $year += 1900;
$time = strftime("%Y-%m-%dT%H:%M", gmtime($time));
print $time, "...";
for(my $i=0; $i < scalar @data; $i++) {
# print "$stations[$i] $time $data[$i]\n";
$in->execute($stations[$i], $time, $data[$i]);
print ".";
}
print " done\n";
}

25
new.pl
View File

@ -4,6 +4,7 @@ use strict;
use diagnostics;
use DBI;
use JSON;
use Date::Manip;
use LWP::UserAgent;
my $ua = LWP::UserAgent->new;
@ -48,7 +49,7 @@ print time, "\t" unless $ENV{R_GPLOT};
my $c = 2;
foreach my $station (@stations) {
print STDERR "[$station] Requesting $base$station\n" if $ENV{R_DEBUG};
my $req = HTTP::Request->new(GET => $base . $station);
my $res = $ua->request($req);
@ -57,23 +58,24 @@ foreach my $station (@stations) {
my $lastupdate;
my $watercourse;
my $datum;
if ($res->is_success and $res->content =~ m~root.station~) {
$res->content =~ m~root.station = (.*);~;
if ($res->is_success and $res->content =~ m~station: {~) {
$res->content =~ m~station: ({.*})~;
my $stationdata = from_json($1);
$res->content =~ m~root.chartData = (.*);~;
$res->content =~ m~chart: (\[.*\])~;
my $leveldata = from_json($1);
$stationname = $stationdata->{'external_name'};
$level = $leveldata->{values}->[-1];
$lastupdate = $stationdata->{value_timestamps}->[-1];
$stationname = $stationdata->{'name'};
$level = $leveldata->[0]->{_};
# $res->content =~ m~Recorded at (.*)~;
$lastupdate = UnixDate($leveldata->[0]->{ts}, "%Y-%m-%dT%T.000Z");
print STDERR "[$station] $stationname is $level at $lastupdate\n" if $ENV{R_DEBUG};
if($ENV{R_GPLOT}) {
$watercourse = $stationdata->{'wiski_river_name'};
$watercourse = $stationdata->{'river'};
$watercourse =~ s~\s*$~~;
$watercourse =~ s~^\s*~~;
$datum = $stationdata->{'wiski_river_name'};
$datum = $stationdata->{'name'};
$datum =~ s~\s*$~~;
$datum =~ s~^\s*~~;
@ -93,7 +95,10 @@ foreach my $station (@stations) {
}
print "\t";
}
} else { print "-\t";}
} else {
print STDERR "Failed to get data for $station\n";
print "-\t";
}
$c++;
}

50
new.plot.pl Executable file
View File

@ -0,0 +1,50 @@
use strict;
use diagnostics;
use DBI;
my $db = DBI->connect("dbi:SQLite:dbname=rivers.db") or die $DBI::errstr;
my $stations_st = $db->prepare("select id, name, watercourse from stations");
$stations_st->execute();
my $stations = $stations_st->fetchall_hashref("id");
my $watercourses = {};
foreach my $station (keys %{$stations}) {
push @{$watercourses->{$stations->{$station}->{watercourse}}}, $station;
}
print <<EOF;
set term png medium size 3200,1200 enhanced font "Helvitica" 14
set datafile missing "-"
set xlabel "Date"
set timefmt "%Y-%m-%dT%H:%M"
set format x "%Y-%m-%d"
set xdata time
set xtics autofreq
set ylabel "River level (m)"
set grid y
set key outside bottom center horizontal
set label 11 left at graph 0,char 1 "Data from Environment Agency processed via mafoo.org.uk" font ",12"
EOF
1;
foreach my $watercourse (keys %{$watercourses}) {
print "\n### $watercourse\n";
my $file = lc $watercourse;
$file =~ s~[^\w\d]+~_~g;
print "set output 'new.out/$file.png'\n";
print "set title '$watercourse'\n";
print "plot \\\n";
my @plots;
foreach my $stationid (@{$watercourses->{$watercourse}}) {
push @plots, qq( "< sqlite3 -init plot.sqliterc rivers.db 'select timestamp, level from levels where stationid = $stationid'" using 1:2 title "$stations->{$stationid}->{name}" with linespoints);
}
print join(", \\\n", @plots), "\n";
}