mafoo.org.uk/rivers
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: mafoo.org.uk:fdcd4e5fe1d44fed5fc8843de9c8b5c6e9f78291
Branches Tags
mafoo.org.uk:master
...
compare: mafoo.org.uk:master
Branches Tags
mafoo.org.uk:master

13 Commits
fdcd4e5fe1 ... master

Author SHA1 Message Date
Matthew Slowe
7977aa96fb add 2024 2024-01-07 17:02:42 +00:00
Matthew Slowe
67d116f570 indentation 2024-01-07 17:02:29 +00:00
Matthew Slowe
8078766a1a disable diagnostics 2024-01-07 17:02:16 +00:00
Matthew Slowe
db674670d1 Updates to copy and link to EA site 2023-10-25 12:53:32 +01:00
Matthew Slowe
50710ba6cb email 2023-10-25 12:51:23 +01:00
Matthew Slowe
4554da48f9 toc 2023-10-25 12:49:15 +01:00
Matthew Slowe
ca483a1986 stacking dashboard 2023-10-25 12:47:43 +01:00
Matthew Slowe
bb8a275163 annual stacking 2023-10-25 11:25:00 +01:00
Matthew Slowe
fe8c836182 ignore tsv, xz, db and out 2023-10-25 11:16:30 +01:00
Matthew Slowe
67915b71dd support exclude column 2023-10-25 11:16:15 +01:00
Matthew Slowe
7888387315 handle error cases from http get 2023-10-25 11:15:55 +01:00
Matthew Slowe
0fb8f35b2d actually ignore the header 2023-10-20 08:27:13 +01:00
Matthew Slowe
39d1a5e328 Use CSV as it's much quicker and easier than scraping HTML :-) 2023-10-20 08:04:04 +01:00
7 changed files with 230 additions and 5 deletions
Show Stats Expand all files Collapse all files

8
.gitignore vendored
View File

@ -1,10 +1,10 @@
env
*.tsv
*.xz
river_level_1135
river_levels_17
*.conf
*.db
out
*.pl

63
csv.pl Executable file
View File

@ -0,0 +1,63 @@
#!/usr/bin/perl -w
$|=1 if $ENV{DEBUG};
use strict;
use diagnostics;
use DBI;
use Text::CSV;
use Date::Manip;
use LWP::UserAgent;
my $ua = LWP::UserAgent->new;
$ua->agent("Munin[river_levels_]");
my @stations = qw(
1129 1135 1133 1143 1132 1137 1134 1145 1139 1131 1130 1144 9274
1086 1085 1073 1091 1071 1076 1069 1066 1067 1072 1074 1083 1064 1090 1089 1088 1087 1078 1080 1063
1030 1120 1121 1122 1125 1126 1127
6074 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6246 6247 6249 6250 6251 6253 6254 6255 6257 6259 6260 6262 6265 6267 6270 6271 6272 6273 6275 6278 6279 6280 6281
6245 6252 6256 6261
6009 6248 6268 6277 6269
6075 6076 6077 6078 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6263 6264 6266 6276 6258 6164 6274
3129 3130 3131 3132 3134 3135 3136 3138 3140 3145 3146 3149 3150 3151 3153
5178 5175 5060 5090 5140 5179 5103
);
my $base = "https://check-for-flooding.service.gov.uk/station-csv/";
my $db = DBI->connect("dbi:SQLite:dbname=rivers.db") or die $DBI::errstr;
my $recordlevel = $db->prepare("REPLACE INTO levels (stationid, timestamp, level) VALUES (?,?,?)") or die $db->errstr;
my $getlastrecord = $db->prepare("SELECT MAX(timestamp) FROM levels WHERE stationid=?") or die $db->errstr;
sub getlatest($) {
my ($id) = @_;
$getlastrecord->execute($id);
my ($lastrecord) = $getlastrecord->fetchrow();
return $lastrecord ? $lastrecord : "0";
}
foreach my $station (@stations) {
print STDERR "[$station] Requesting $base$station\n" if $ENV{R_DEBUG};
my $req = HTTP::Request->new(GET => $base . $station);
my $res = $ua->request($req);
if($res->is_success) {
my $content = $res->content;
my $last = getlatest($station);
print "[$station] Last record was $last\n" if $ENV{R_DEBUG};
foreach my $line (sort split(/^/m, $content)) {
next if $line =~ /Timestamp/;
chomp($line);
my ($ts, $level) = split(/,/, $line);
if($ts le $last) {
print "[$station] Skipping $level at $ts\n" if $ENV{R_DEBUG};
} else {
print "[$station] Recording $level at $ts\n" if $ENV{R_DEBUG};
$recordlevel->execute($station, $ts, $level);
}
}
} else {
warn "Skipped $station as request failed: $res->status_line";
}
}

2
new.plot.pl
View File

@ -4,7 +4,7 @@ use DBI;
my $db = DBI->connect("dbi:SQLite:dbname=rivers.db") or die $DBI::errstr;
my $stations_st = $db->prepare("select id, name, watercourse from stations");
my $stations_st = $db->prepare("select id, name, watercourse from stations where exclude<>'yes'");
$stations_st->execute();
my $stations = $stations_st->fetchall_hashref("id");

44
out/stacked.html Normal file
View File

@ -0,0 +1,44 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>Stacked river levels for the Great Stour around Canterbury, Kent</title>
</head>
<body>
<h1>Stacked river levels for the Great Stour around Canterbury, Kent</h1>
<ul>
<li><a href="#wye">Wye</a></li>
<li><a href="#chartham">Chartham</a></li>
<li><a href="#westgategardens">Westgate Gardens</a></li>
<li><a href="#vauxhallbridge">Vauxhall Bridge</a></li>
</ul>
<h2>Background</h2>
<p>
The charts below show the annual levels of the Great Stour at various locations between Ashford and Canterbury in Kent.
</p>
<p>
Data collection began in 2014 for <a href="https://check-for-flooding.service.gov.uk/station/1145">Wye</a>, <a href="https://check-for-flooding.service.gov.uk/station/1135">Chartham</a>, and <a href="https://check-for-flooding.service.gov.uk/station/1143">Vauxhall bridge</a>. <a href="https://check-for-flooding.service.gov.uk/station/9274">Westgate Gardens</a> is a newer monitoring station and data is available back from 2020. There are a few periods of time where data is unavailable normally due to changes in the upstream data availability.
</p>
<p style="font-style: italic">
Data has been gathered from the <a href="https://check-for-flooding.service.gov.uk">Environment Agency in England</a> and is available under the the <a href="https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/">Open Government Licence v3.0</a>, except where otherwise stated.
</p>
<p>
If you'd like a copy of the underlying data then please <a href="mailto:foo+rivers@mafoo.org.uk">drop me an email</a>.
</p>
<a id="wye"><h2>Wye</h2></a>
<img src="canterbury-1145-stacked.png" />
<a id="chartham"><h2>Chartham</h2></a>
<img src="canterbury-1135-stacked.png" />
<a id="westgategardens"><h2>Westgate Gardens, Canterbury</h2></a>
<img src="canterbury-9274-stacked.png" />
<a id="vauxhallbridge"><h2>Vauxhall Bridge, Canterbury</h2></a>
<img src="canterbury-1143-stacked.png" />
</body>
</html>

9
stack-build.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/bash
set -e
for n in 1143 1145 1135 9274
do
perl stack-years.pl $n >$n-pivot.txt
done
time gnuplot stack-by-year.gnuplot

73
stack-by-year.gnuplot Normal file
View File

@ -0,0 +1,73 @@
set term png medium size 6400,1200 enhanced font "Helvitica" 14
set datafile missing "-"
set xlabel "Date"
set timefmt "%s"
set format x "%Y-%m-%d"
# set format x "%Y-%m-%d %H:%M:%S"
set xdata time
# set xtics rotate by 45 offset -0.8,-1.8
set xtics autofreq
set ylabel "River level (m)"
set grid y
set key outside bottom center horizontal
set label 11 left at graph 0,char 1 "Data from Environment Agency processed via mafoo.org.uk" font ",12"
### Canterbury
set term png medium size 3600,800 enhanced font "Helvitica" 14
set timefmt "%Y-%m-%dT%H:%M"
set xrange ['2000-01-01 00:00:00':'2000-12-31 23:59:59']
set output "out/canterbury-1143-stacked.png"
set title "River Levels around at Vauxhall Bridge, Great Stour"
plot \
"1143-pivot.txt" using 1:2 with linespoints title "2014", \
"1143-pivot.txt" using 1:3 with linespoints title "2015", \
"1143-pivot.txt" using 1:4 with linespoints title "2016", \
"1143-pivot.txt" using 1:5 with linespoints title "2017", \
"1143-pivot.txt" using 1:6 with linespoints title "2018", \
"1143-pivot.txt" using 1:7 with linespoints title "2019", \
"1143-pivot.txt" using 1:8 with linespoints title "2020", \
"1143-pivot.txt" using 1:9 with linespoints title "2021", \
"1143-pivot.txt" using 1:10 with linespoints title "2022", \
"1143-pivot.txt" using 1:11 with linespoints title "2023"
set output "out/canterbury-1145-stacked.png"
set title "River Levels around at Wye, Great Stour"
plot \
"1145-pivot.txt" using 1:2 with linespoints title "2014", \
"1145-pivot.txt" using 1:3 with linespoints title "2015", \
"1145-pivot.txt" using 1:4 with linespoints title "2016", \
"1145-pivot.txt" using 1:5 with linespoints title "2017", \
"1145-pivot.txt" using 1:6 with linespoints title "2018", \
"1145-pivot.txt" using 1:7 with linespoints title "2019", \
"1145-pivot.txt" using 1:8 with linespoints title "2020", \
"1145-pivot.txt" using 1:9 with linespoints title "2021", \
"1145-pivot.txt" using 1:10 with linespoints title "2022", \
"1145-pivot.txt" using 1:11 with linespoints title "2023"
set output "out/canterbury-1135-stacked.png"
set title "River Levels around at Chartham, Great Stour"
plot \
"1135-pivot.txt" using 1:2 with linespoints title "2014", \
"1135-pivot.txt" using 1:3 with linespoints title "2015", \
"1135-pivot.txt" using 1:4 with linespoints title "2016", \
"1135-pivot.txt" using 1:5 with linespoints title "2017", \
"1135-pivot.txt" using 1:6 with linespoints title "2018", \
"1135-pivot.txt" using 1:7 with linespoints title "2019", \
"1135-pivot.txt" using 1:8 with linespoints title "2020", \
"1135-pivot.txt" using 1:9 with linespoints title "2021", \
"1135-pivot.txt" using 1:10 with linespoints title "2022", \
"1135-pivot.txt" using 1:11 with linespoints title "2023"
set output "out/canterbury-9274-stacked.png"
set title "River Levels around at Westgate Gardens, Great Stour"
plot \
"9274-pivot.txt" using 1:2 with linespoints title "2020", \
"9274-pivot.txt" using 1:3 with linespoints title "2021", \
"9274-pivot.txt" using 1:4 with linespoints title "2022", \
"9274-pivot.txt" using 1:5 with linespoints title "2023"

36
stack-years.pl Normal file
View File

@ -0,0 +1,36 @@
#!/usr/bin/perl -w
use strict;
# use diagnostics;
use DBI;
use POSIX;
use Data::Dumper;
my $station = shift @ARGV or die;
my $db = DBI->connect("dbi:SQLite:dbname=rivers.db") or die $DBI::errstr;
my $years_st = $db->prepare("select distinct substr(timestamp, 1, 4) from levels where stationid=?");
my $getdata_st = $db->prepare("select strftime('2000-%m-%dT%H:%M:%S.000Z', timestamp) as t, level from levels where stationid = ? and timestamp like ? || '-%'");
$years_st->execute($station);
my @years = @{$years_st->fetchall_arrayref};
my %data;
foreach my $y_ref (@years) {
my $y = $y_ref->[0];
$getdata_st->execute($station, $y);
while(my $row = $getdata_st->fetchrow_arrayref) {
$data{$row->[0]}{$y} = $row->[1];
}
}
# print Dumper(\%data);
foreach my $ts (sort keys %data) {
print "$ts\t";
foreach my $y_ref (@years) {
my $y = $y_ref->[0];
print $data{$ts}{$y} || '-', "\t";
}
print "\n";
}