This is a small Jupyter notebook to show how one might consume the data provided by AgReFed @ Dale. It provides some simple plotting examples using plotly and hvPlot.
It is just javascript suitable for cutting and pasting directly into a browser and a bit of Python.
You can download the actual notebook from: https://webapps.plantenergy.uwa.edu.au/agrefed_dale/static/plotly.ipynb (much more fun!)
Hopefully you can get an idea of how easy it is to incorporate AgReFed @ Dale data into your
own investigations.
see https://plot.ly/javascript/reference/ for a reference to plotly.js
The plotly javascript is at https://cdn.plot.ly/plotly-latest.min.js (all 6MB of it!)
%%javascript
// ask for plotly and give it a name "plotly"
requirejs.config({
paths: {
plotly: 'https://cdn.plot.ly/plotly-latest.min'
}
});
// helper function
window.plotly = (elem, plots, layout) => {
const div = document.createElement('DIV')
elem.append(div)
// ask requirejs to call us when plotly is loaded
requirejs(['plotly'], Plotly => Plotly.newPlot(div, plots, layout))
}
%%javascript
window.Dale = 'https://webapps.plantenergy.uwa.edu.au/agrefed_dale'
Some helper functions
%%javascript
// list of objects to an object of lists
window.unzip = (arr, ...keys) => {
var o = {};
keys.forEach(k => o[k]= []);
arr.forEach(a => {
keys.forEach(k => o[k].push(a[k]))
});
return o;
}
// show failure
window.jqfail = (ele, xhr) => {
const div = $('<div/>');
div.css({"background-color": "orange"})
div.text(xhr.responseText || xhr.statusText || 'unknown error')
ele.append(div);
}
window.fetch_fail = (ele, ...args) => {
args = args.map(x => x.toString()).join(',')
const div = document.createElement('DIV');
div.textContent = `${args}`
div.style.backgroundColor = 'orange'
ele.append(div)
}
fetch is an ajax method available in all(most) modern browsers...
Let's get some site weather data!
Note that in the %%javascript notebook context element is a jQuery object representing the output text area of the
current cell. This way we can avoid creating random id for div elements for plotly to find.
%%javascript
const plot1 = element;
// we only want these columns....
const cols1 = ['datetime', 'soilt', 'airt', 'AT_60cm', 'AT_30cm']
// our query per_page:0 mean *all* data
var params = { per_page:0, datetime: "2018-02-17/P20D", select:cols1.join(',') }
var url = new URL(Dale + '/WeatherSite')
url.search = new URLSearchParams(params)
fetch(url).then(resp => resp.json())
.then(json => unzip(json.items, ...cols1))
.then(({datetime, ...rest}) => {
// turn data into plotly plots.
const traces = Object.entries(rest).map(([name, y]) => {
return {
x: datetime,
y: y,
type: 'scatter',
name: name
}
});
plotly(plot1, traces, {width:900});
}).catch((...args) => fetch_fail(plot1, ...args))
We always have jQuery available in a notebook
%%javascript
const plot2 = element;
const cols2 = ['datetime', 'soilt', 'airt']
var params = {per_page:0, datetime: "2018-02-17/2018-04-17", select: cols2.join(',')}
$.get(Dale + '/WeatherSite', params).done(json => {
const {datetime, ...rest} = unzip(json.items, ...cols2);
const traces = Object.entries(rest).map(([name, y]) => {
return {
x: datetime,
y: y,
type: 'scatter',
name: name
}
});
plotly(plot2, traces, {width:900});
}).fail((xhr) => jqfail(plot2, xhr))
Usually you want to compare a column to a value e.g. airt > 38. But you can
also compare other columns if you "quote" the value with backticks: e.g. airt > `soilt`.
Here we plot all places where the air temperature was greater than the soil temperature.
We have to target the /query/suba endpoint which understands SUBA query language. See
here.
%%javascript
const plot3 = element;
const cols3 = ['datetime', 'soilt', 'airt']
var params = {per_page:0, query: "airt > `soilt`", entity_name: "WeatherSite", select: cols3.join(',')}
$.post(Dale + '/suba/query', params).done(json => {
const {datetime, ...rest} = unzip(json.items, ...cols3);
const traces = Object.entries(rest).map(([name, y]) => {
return {
x: datetime,
y: y,
type: 'scatter',
mode: 'markers',
name: name
}
});
plotly(plot3, traces, {width:900});
}).fail((xhr) => jqfail(plot3, xhr))
Let's look at how crop yield varied across plots
%%javascript
const plot4 = element;
const cols4 = ['plot_barcode', 'latitude', 'longitude', 'crop_yield']
$.get(Dale + '/Plot', {per_page: 0, select: cols4.join(',')}).done(page => {
const {plot_barcode, latitude, longitude, crop_yield} = unzip(page.items, ...cols4);
const traces = [
{
y: latitude,
x: longitude,
type: 'scatter',
mode: 'markers',
text: page.items.map(r => `${r.plot_barcode}: ${r.crop_yield}`),
marker : {
size:crop_yield.map(y => 2*y),
color:crop_yield,
colorscale: 'Viridis',
colorbar: {
title: 'Crop Yield'
}
}
}]
const axis = {
tickformat: ".3f",
ticksuffix: "°",
}
const layout = {
height: 750,
width: 900,
title: 'plot yield vs location',
xaxis : {
title: "longitude", ...axis
},
yaxis : {
title: "latitude", ...axis
}
}
plotly(plot4, traces, layout);
}).fail((xhr) => jqfail(plot4, xhr))
... or maybe you just want the data
Let's get a nice json renderer
%%javascript
requirejs.config({
paths: {
renderjson: 'https://cdn.jsdelivr.net/npm/[email protected]/renderjson.min'
}
});
window.renderjson = (elem, json, level=2) => {
requirejs(['renderjson'], r => {
r.renderjson.set_show_to_level(level);
elem.append(r.renderjson(json))
})
}
... and give it some colour
%%html
<style type="text/css">
pre.renderjson .string {
color: #007700;
}
pre.renderjson .key {
color: #000077;
}
</style>
Now we can just look over the data.
Let's get a page of data
Notice the weird next_url? Just "get" that endpoint and you'll have the next page of data from
your query (nice hey!). You can step forward (and back!) through the pages easily.
%%javascript
const plot5 = element;
var cols5 = ['datetime', 'soilt', 'airt']
var params = {per_page:5, query: "airt > `soilt`", entity_name: "WeatherSite", select: cols5.join(',')}
$.post(Dale + '/suba/query', params).done(json => {
renderjson(plot5, json, 3)
}).fail((xhr) => jqfail(plot5, xhr))
All the metadata about the data can be found from the OAS schema.
In particular components.schemas.{Table}.properties gives details on the available data names.
You can also find all the information here.
%%javascript
const plot6 = element;
$.get(Dale + '/swagger.json').done(json => {
renderjson(plot6, json, 2)
}).fail((xhr) => jqfail(plot6, xhr))
It's just as easy from Python
import requests
import pandas as pd
Dale = 'https://webapps.plantenergy.uwa.edu.au/agrefed_dale'
params = {
'per_page': 20,
'query': "airt > `soilt`",
'entity_name': "WeatherSite",
'ordering' : '<datetime' # order by datetime ascending
}
data = requests.post(Dale + '/suba/query', data=params).json()
df = pd.DataFrame(data['items'])
df.datetime = pd.to_datetime(df.datetime) # datetimes are always a problem
df
keep re-running this cell to step through the pages
data = requests.get(data['next_url']).json()
df = pd.DataFrame(data['items'])
df.datetime = pd.to_datetime(df.datetime)
df
Use headers= {"accept": "text/csv"} to get a CSV version of the data
from io import StringIO
cols = ['datetime', 'soilt', 'airt']
params = {
'per_page': 0,
'select': ','.join(cols),
'datetime': "2018-02-17/P20D",
}
data = requests.get(Dale + '/WeatherSite',
params,
headers={'accept': 'text/csv'})
df = pd.read_csv(StringIO(data.text))
A good way to plot datafames is hvPlot.
It adds a hvPlot method to the dataframe object. (You might want to uncomment the line below
before proceeding)
#!pip install hvplot
# *or* if that fails with permission errors
#!pip install --user hvplot
import hvplot.pandas
#%%output filename="plot" fig="png"
df.datetime = pd.to_datetime(df.datetime)
df.set_index('datetime').hvplot(width=900)
cols = ['plot_barcode', 'latitude', 'longitude', 'crop_yield']
resp = requests.get(Dale + '/Plot', {'per_page': 0, 'select': ','.join(cols)})
df = pd.DataFrame(resp.json()['items'])
df.hvplot.scatter("longitude",
"latitude",
s='crop_yield',
c='crop_yield',
scale=3,
cmap='viridis',
height=500,
width=700,
colorbar=True)