Inline Example of Local Expert ‘Optimal Interpolation’ on Satellite Data
Using Colab? Then clone and install
[1]:
try:
import google.colab
IN_COLAB = True
except:
IN_COLAB = False
if IN_COLAB:
import subprocess
import os
import re
# change to working directory
work_dir = "/content"
assert os.path.exists(work_dir), f"workspace directory: {work_dir} does not exist"
os.chdir(work_dir)
# clone repository
command = "git clone https://github.com/CPOMUCL/GPSat.git"
result = subprocess.run(command.split(), capture_output=True, text=True)
print(result.stdout)
repo_dir = os.path.join(work_dir, "GPSat")
print(f"changing directory to: {repo_dir}")
os.chdir(repo_dir)
# exclude certain requirements if running on colab - namely avoid installing/upgrading tensorflow
new_req = []
with open(os.path.join(repo_dir, "requirements.txt"), "r") as f:
for line in f.readlines():
# NOTE: here also removing numpy requirement
if re.search("^tensorflow|^numpy", line):
new_req.append("#" + line)
else:
new_req.append(line)
# create a colab specific requirements file
with open(os.path.join(repo_dir, "requirements_colab.txt"), "w") as f:
f.writelines(new_req)
# install the requirements
command = "pip install -r requirements_colab.txt"
with subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) as proc:
# Stream the standard output in real-time
for line in proc.stdout:
print(line, end='')
# install the GPSat pacakge in editable mode
command = "pip install -e ."
with subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) as proc:
# Stream the standard output in real-time
for line in proc.stdout:
print(line, end='')
Import Packages
[2]:
import os
import re
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from global_land_mask import globe
from GPSat import get_data_path, get_parent_path
from GPSat.dataprepper import DataPrep
from GPSat.dataloader import DataLoader
from GPSat.utils import stats_on_vals, WGS84toEASE2, EASE2toWGS84, cprint, grid_2d_flatten, get_weighted_values
from GPSat.local_experts import LocalExpertOI, get_results_from_h5file
from GPSat.plot_utils import plot_wrapper, plot_pcolormesh, get_projection, plot_pcolormesh_from_results_data, plot_hyper_parameters
from GPSat.postprocessing import smooth_hyperparameters
2025-10-02 22:18:28.650637: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
2025-10-02 22:18:28.650677: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-10-02 22:18:28.651576: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2025-10-02 22:18:28.656852: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2025-10-02 22:18:29.662207: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
Parameters
[3]:
# NOTE: there are parameters values that are set inline in the cells below
# lat,lon center (origin) used for converting between WGS84 and EASE2 projections
lat_0 = 90
lon_0 = 0
# expert location parameters
# spacing between experts (laid out on a grid), in meters
expert_spacing = 200_000
# range of experts, from origin, in meters
# expert_x_range = [-750_000.0, 1000_000.0]
# expert_y_range = [-500_000.0, 1250_000.0]
expert_x_range = [-500_000, 500_000]
expert_y_range = [-500_000, 500_000]
# prediction spacing
# (below predictions same range as experts)
pred_spacing = 5_000
# model parameters
# Set training and inference radius
# - distance observations need to be away from expert locations to be included in training
training_radius = 300_000 # 300km
# - distance prediction locations need to be away from expert locations in order of predictions to be made
inference_radius = 200_000 # 200km
# plotting
# extent = [lon min, lat max, lat min, lat max]
extent = [-180, 180, 60, 90]
# which projection to use: "north" or "south"
projection = "north"
read in raw data
add each key in col_func as a column, using a specified function + arguments values are unpacked and passed to GPSat.utils.config_func
[4]:
df = DataLoader.read_flat_files(file_dirs=get_data_path("example"),
file_regex="_RAW\.csv$",
col_funcs={
"source": {
"func": lambda x: re.sub('_RAW.*$', '', os.path.basename(x)),
"filename_as_arg": True
}
})
# convert lon, lat, datetime to x, y, t - to be used as the coordinate space
df['x'], df['y'] = WGS84toEASE2(lon=df['lon'], lat=df['lat'], lat_0=lat_0, lon_0=lon_0)
df['t'] = df['datetime'].values.astype("datetime64[D]").astype(float)
----------------------------------------------------------------------------------------------------
reading files from:
/home/runner/work/GPSat/GPSat/data/example/
that match regular expression: _RAW\.csv$
'read_from_multiple_files': 0.728 seconds
stats on data
[5]:
print("*" * 20)
print("summary / stats table on metric (use for trimming)")
val_col = 'z'
vals = df[val_col].values
stats_df = stats_on_vals(vals=vals, name=val_col,
qs=[0.01, 0.05] + np.arange(0.1, 1.0, 0.1).tolist() + [0.95, 0.99])
print(stats_df)
********************
summary / stats table on metric (use for trimming)
'stats_on_vals': 0.237 seconds
z
measure z
size 1174848
num_not_nan 1174848
num_inf 0
min -16.7965
mean 0.128416
max 16.7093
std 0.186566
skew -10.912819
kurtosis 890.268297
q0.010 -0.3732
q0.050 -0.1387
q0.100 -0.0485
q0.200 0.0322
q0.300 0.0767
q0.400 0.1101
q0.500 0.1394
q0.600 0.1686
q0.700 0.2011
q0.800 0.2424
q0.900 0.3067
q0.950 0.3632
q0.990 0.4775
visualise data
[6]:
# plot observations and histogram
fig, stats_df = plot_wrapper(plt_df=df,
val_col=val_col,
max_obs=500_000,
vmin_max=[-0.1, 0.5],
projection=projection,
extent=extent)
plt.show()
'stats_on_vals': 0.287 seconds
there were too many points 1174848>max_obs: 500000
selecting 42.55% (499903) points at random for raw data plot
plotting pcolormesh...
'plot_pcolormesh': 0.138 seconds
plotting hist (using all data)...
'plot_hist': 1.901 seconds
'plot_wrapper': 2.521 seconds
bin raw data
bin by date, source - returns a DataSet
[7]:
bin_ds = DataPrep.bin_data_by(df=df.loc[(df['z'] > -0.35) & (df['z'] < 0.65)],
by_cols=['t', 'source'],
val_col=val_col,
x_col='x',
y_col='y',
grid_res=50_000,
x_range=[-4500_000.0, 4500_000.0],
y_range=[-4500_000.0, 4500_000.0])
# convert bin data to DataFrame
# - removing all the nans that would be added at grid locations away from data
bin_df = bin_ds.to_dataframe().dropna().reset_index()
'bin_data_by': 1.691 seconds
plot binned data
[8]:
# this will plot all observations, some on top of each other
bin_df['lon'], bin_df['lat'] = EASE2toWGS84(bin_df['x'], bin_df['y'],
lat_0=lat_0, lon_0=lon_0)
# plot observations and histogram
fig, stats_df = plot_wrapper(plt_df=bin_df,
val_col=val_col,
max_obs=500_000,
vmin_max=[-0.1, 0.5],
projection=projection,
extent=extent)
plt.show()
'stats_on_vals': 0.007 seconds
plotting pcolormesh...
'plot_pcolormesh': 0.020 seconds
plotting hist (using all data)...
'plot_hist': 0.201 seconds
'plot_wrapper': 0.273 seconds
expert locations
on evenly spaced grid
[9]:
xy_grid = grid_2d_flatten(x_range=expert_x_range,
y_range=expert_y_range,
step_size=expert_spacing)
# store in dataframe
eloc = pd.DataFrame(xy_grid, columns=['x', 'y'])
# add a time coordinate
eloc['t'] = np.floor(df['t'].mean())
plot expert locations
[10]:
eloc['lon'], eloc['lat'] = EASE2toWGS84(eloc['x'], eloc['y'],
lat_0=lat_0, lon_0=lon_0)
fig = plt.figure(figsize=(12, 12))
ax = fig.add_subplot(1, 1, 1, projection=get_projection(projection))
plot_pcolormesh(ax=ax,
lon=eloc['lon'],
lat=eloc['lat'],
plot_data=eloc['t'],
title="expert locations",
scatter=True,
s=20,
fig=fig,
extent=extent)
plt.tight_layout()
plt.show()
'plot_pcolormesh': 0.021 seconds
prediction locations
[11]:
pred_xy_grid = grid_2d_flatten(x_range=expert_x_range,
y_range=expert_y_range,
step_size=pred_spacing)
# store in dataframe
# NOTE: the missing 't' coordinate will be determine by the expert location
# - alternatively the prediction location can be specified
ploc = pd.DataFrame(pred_xy_grid, columns=['x', 'y'])
ploc['lon'], ploc['lat'] = EASE2toWGS84(ploc['x'], ploc['y'],
lat_0=lat_0, lon_0=lon_0)
# identify if a position is in the ocean (water) or not
ploc["is_in_ocean"] = globe.is_ocean(ploc['lat'], ploc['lon'])
# keep only prediction locations in ocean
ploc = ploc.loc[ploc["is_in_ocean"]]
plot prediction locations
[12]:
fig = plt.figure(figsize=(12, 12))
ax = fig.add_subplot(1, 1, 1, projection=get_projection(projection))
plot_pcolormesh(ax=ax,
lon=ploc['lon'],
lat=ploc['lat'],
plot_data=np.full(len(ploc), 1.0), # np.arange(len(ploc)),
title="prediction locations",
scatter=True,
s=0.1,
# fig=fig,
extent=extent)
plt.tight_layout()
plt.show()
'plot_pcolormesh': 0.014 seconds
configurations:
[13]:
# observation data
data = {
"data_source": bin_df,
"obs_col": "z",
"coords_col": ["x", "y", "t"],
# selection criteria used for each local expert
"local_select": [
{
"col": "t",
"comp": "<=",
"val": 4
},
{
"col": "t",
"comp": ">=",
"val": -4
},
{
"col": [
"x",
"y"
],
"comp": "<",
"val": training_radius
}
]
}
# local expert locations
local_expert = {
"source": eloc
}
# model
model = {
"oi_model": "GPflowGPRModel",
"init_params": {
# scale (divide) coordinates
"coords_scale": [50000, 50000, 1],
# can specify initial parameters values for model:
# "noise_variance": 0.10,
# "kernel_kwargs": {
# "lengthscales": [2.0, 2.0, 1.0],
# "variance": 0.05
# }
},
# keyword arguments to be passed into each model/local expert's optimise_parameters method
"optim_kwargs": {
# parameters to be fixed (not trainable)
# "fixed_params": ["likelihood_variance"]
},
"constraints": {
# lengthscales - same order coord_col (see data)
# - given in unscaled units
"lengthscales": {
"low": [1e-08, 1e-08, 1e-08],
"high": [600000, 600000, 9]
},
"likelihood_variance": {
"low": 0.00125,
"high": 0.01
}
}
}
# prediction locations
pred_loc = {
"method": "from_dataframe",
"df": ploc,
"max_dist": inference_radius
}
Local Expert OI
if process falls over here when calling run(), try: Runtime -> “Restart and run all”
[14]:
locexp = LocalExpertOI(expert_loc_config=local_expert,
data_config=data,
model_config=model,
pred_loc_config=pred_loc)
# run optimal interpolation
# - no predictions locations supplied
store_path = get_parent_path("results", "inline_example.h5")
# for the purposes of a simple example, if store_path exists: delete it
if os.path.exists(store_path):
cprint(f"removing: {store_path}", "FAIL")
os.remove(store_path)
# run optimal interpolation
locexp.run(store_path=store_path,
optimise=True,
check_config_compatible=False)
'data_select': 0.000 seconds
'load': 0.000 seconds
in json_serializable - key: 'data_source' has value DataFrame/Series, but is too long: 22522 > 100
storing as str
in json_serializable - key: 'df' has value DataFrame/Series, but is too long: 40000 > 100
storing as str
---------
storing expert locations in 'expert_locs' table
exception occurred: 'No object named expert_locs in the file'
will now close object
---------
dropping expert locations that already exists in 'run_details' table
exception occurred: 'No object named run_details in the file'
will now close object
--------------------------------------------------
1 / 25
current local expert:
x y t lon lat
0 -400000.0 -400000.0 18326.0 -45.0 84.933616
'data_select': 0.000 seconds
'load': 0.001 seconds
'local_data_select': 0.007 seconds
number obs: 402
setting lengthscales to: [1. 1. 1.]
'__init__': 0.061 seconds
'get_parameters': 0.006 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.025 seconds
'optimise_parameters': 2.640 seconds
'get_parameters': 0.005 seconds
parameters:
lengthscales: array([5.18518252, 3.21964264, 8.99998908])
kernel_variance: 0.015251959221648119
likelihood_variance: 0.003326332353056036
'predict': 0.167 seconds
total run time : 3.25 seconds
--------------------------------------------------
2 / 25
current local expert:
x y t lon lat
1 -200000.0 -400000.0 18326.0 -26.565051 85.9952
'local_data_select': 0.006 seconds
number obs: 538
setting lengthscales to: [1. 1. 1.]
'__init__': 0.025 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 2.542 seconds
'get_parameters': 0.005 seconds
parameters:
lengthscales: array([3.77522845, 2.17856697, 8.9999992 ])
kernel_variance: 0.013369622401462052
likelihood_variance: 0.0020117719824816445
'predict': 0.278 seconds
total run time : 3.21 seconds
--------------------------------------------------
3 / 25
current local expert:
x y t lon lat
2 0.0 -400000.0 18326.0 0.0 86.418156
'local_data_select': 0.006 seconds
number obs: 519
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 1.899 seconds
'get_parameters': 0.005 seconds
parameters:
lengthscales: array([4.39952074, 7.18064058, 8.99999806])
kernel_variance: 0.009118927918986074
likelihood_variance: 0.0018970594765749756
'predict': 0.272 seconds
total run time : 2.56 seconds
--------------------------------------------------
4 / 25
current local expert:
x y t lon lat
3 200000.0 -400000.0 18326.0 26.565051 85.9952
'local_data_select': 0.007 seconds
number obs: 525
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 1.730 seconds
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([6.84405356, 6.23994376, 8.99981492])
kernel_variance: 0.014942999337311334
likelihood_variance: 0.00166630824315081
'predict': 0.261 seconds
total run time : 2.38 seconds
--------------------------------------------------
5 / 25
current local expert:
x y t lon lat
4 400000.0 -400000.0 18326.0 45.0 84.933616
'local_data_select': 0.006 seconds
number obs: 409
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 1.032 seconds
'get_parameters': 0.005 seconds
parameters:
lengthscales: array([6.71046897, 4.73273014, 8.99996646])
kernel_variance: 0.013183663218678301
likelihood_variance: 0.0016308604144238922
'predict': 0.189 seconds
total run time : 1.63 seconds
--------------------------------------------------
6 / 25
current local expert:
x y t lon lat
5 -400000.0 -200000.0 18326.0 -63.434949 85.9952
'local_data_select': 0.006 seconds
number obs: 566
setting lengthscales to: [1. 1. 1.]
'__init__': 0.025 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 2.082 seconds
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([6.87515704, 9.42140437, 3.38399959])
kernel_variance: 0.023405454310086084
likelihood_variance: 0.0024454365638836477
'predict': 0.287 seconds
total run time : 2.77 seconds
--------------------------------------------------
7 / 25
current local expert:
x y t lon lat
6 -200000.0 -200000.0 18326.0 -45.0 87.467477
'local_data_select': 0.007 seconds
number obs: 548
setting lengthscales to: [1. 1. 1.]
'__init__': 0.025 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 1.971 seconds
'get_parameters': 0.005 seconds
parameters:
lengthscales: array([ 6.46135277, 11.99999999, 8.99997935])
kernel_variance: 0.028392606031534606
likelihood_variance: 0.0018270093804950596
'predict': 0.339 seconds
total run time : 2.73 seconds
--------------------------------------------------
8 / 25
current local expert:
x y t lon lat
7 0.0 -200000.0 18326.0 0.0 88.209314
'local_data_select': 0.006 seconds
number obs: 501
setting lengthscales to: [1. 1. 1.]
'__init__': 0.025 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 1.498 seconds
'get_parameters': 0.005 seconds
parameters:
lengthscales: array([6.08708649, 5.81881187, 8.99999351])
kernel_variance: 0.0099106042211078
likelihood_variance: 0.0015218385845996042
'predict': 0.302 seconds
total run time : 2.21 seconds
--------------------------------------------------
9 / 25
current local expert:
x y t lon lat
8 200000.0 -200000.0 18326.0 45.0 87.467477
'local_data_select': 0.007 seconds
number obs: 525
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 2.204 seconds
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([8.09284798, 4.33060048, 8.18147106])
kernel_variance: 0.0130109471618823
likelihood_variance: 0.0012717766991325442
'predict': 0.312 seconds
total run time : 2.91 seconds
--------------------------------------------------
10 / 25
current local expert:
x y t lon lat
9 400000.0 -200000.0 18326.0 63.434949 85.9952
'local_data_select': 0.006 seconds
number obs: 569
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 2.720 seconds
'get_parameters': 0.005 seconds
parameters:
lengthscales: array([11.92767324, 6.49464432, 8.72242301])
kernel_variance: 0.01397106614006652
likelihood_variance: 0.0016706777074234976
'predict': 0.305 seconds
SAVING RESULTS TO TABLES:
run_details
preds
lengthscales
kernel_variance
likelihood_variance
total run time : 3.57 seconds
--------------------------------------------------
11 / 25
current local expert:
x y t lon lat
10 -400000.0 0.0 18326.0 -90.0 86.418156
'local_data_select': 0.006 seconds
number obs: 591
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 3.305 seconds
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([ 7.24233279, 11.99999 , 3.80743502])
kernel_variance: 0.01955317863585096
likelihood_variance: 0.0023658548443428636
'predict': 0.305 seconds
total run time : 4.02 seconds
--------------------------------------------------
12 / 25
current local expert:
x y t lon lat
11 -200000.0 0.0 18326.0 -90.0 88.209314
'local_data_select': 0.007 seconds
number obs: 516
setting lengthscales to: [1. 1. 1.]
'__init__': 0.025 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
**********
optimization failed!
'optimise_parameters': 3.289 seconds
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([11.99998867, 7.41165492, 4.68893604])
kernel_variance: 0.028614113822342774
likelihood_variance: 0.0018064900415264249
'predict': 0.297 seconds
total run time : 3.98 seconds
--------------------------------------------------
13 / 25
current local expert:
x y t lon lat
12 0.0 0.0 18326.0 0.0 90.0
'local_data_select': 0.006 seconds
number obs: 522
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 2.029 seconds
'get_parameters': 0.005 seconds
parameters:
lengthscales: array([11.86292071, 11.82772079, 8.99999452])
kernel_variance: 0.01049216592801527
likelihood_variance: 0.0012500000848738267
'predict': 0.318 seconds
total run time : 2.75 seconds
--------------------------------------------------
14 / 25
current local expert:
x y t lon lat
13 200000.0 0.0 18326.0 90.0 88.209314
'local_data_select': 0.007 seconds
number obs: 501
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 2.032 seconds
'get_parameters': 0.005 seconds
parameters:
lengthscales: array([10.8804787 , 5.80152815, 8.99989684])
kernel_variance: 0.014735868830929161
likelihood_variance: 0.0012912240585831252
'predict': 0.299 seconds
total run time : 2.72 seconds
--------------------------------------------------
15 / 25
current local expert:
x y t lon lat
14 400000.0 0.0 18326.0 90.0 86.418156
'local_data_select': 0.007 seconds
number obs: 564
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 2.316 seconds
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([9.75902518, 6.26796594, 8.99997672])
kernel_variance: 0.0082802270532373
likelihood_variance: 0.0013510427249318695
'predict': 0.289 seconds
total run time : 3.01 seconds
--------------------------------------------------
16 / 25
current local expert:
x y t lon lat
15 -400000.0 200000.0 18326.0 -116.565051 85.9952
'local_data_select': 0.006 seconds
number obs: 556
setting lengthscales to: [1. 1. 1.]
'__init__': 0.025 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
**********
optimization failed!
'optimise_parameters': 3.241 seconds
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([ 7.10995042, 11.99996794, 3.40268367])
kernel_variance: 0.020404770971216006
likelihood_variance: 0.001968874233339219
'predict': 0.277 seconds
total run time : 3.93 seconds
--------------------------------------------------
17 / 25
current local expert:
x y t lon lat
16 -200000.0 200000.0 18326.0 -135.0 87.467477
'local_data_select': 0.006 seconds
number obs: 556
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
**********
optimization failed!
'optimise_parameters': 3.282 seconds
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([11.99993682, 11.99999032, 3.79106325])
kernel_variance: 0.0217417925672742
likelihood_variance: 0.0017348665792717552
'predict': 0.329 seconds
total run time : 4.04 seconds
--------------------------------------------------
18 / 25
current local expert:
x y t lon lat
17 0.0 200000.0 18326.0 180.0 88.209314
'local_data_select': 0.007 seconds
number obs: 513
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 1.687 seconds
'get_parameters': 0.005 seconds
parameters:
lengthscales: array([7.82064232, 7.58606593, 8.99999999])
kernel_variance: 0.008810638115569262
likelihood_variance: 0.0012500000816909565
'predict': 0.309 seconds
total run time : 2.40 seconds
--------------------------------------------------
19 / 25
current local expert:
x y t lon lat
18 200000.0 200000.0 18326.0 135.0 87.467477
'local_data_select': 0.006 seconds
number obs: 532
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 1.697 seconds
'get_parameters': 0.005 seconds
parameters:
lengthscales: array([ 7.46143747, 11.99998225, 8.99999987])
kernel_variance: 0.007188212009182438
likelihood_variance: 0.0014107074481697094
'predict': 0.320 seconds
total run time : 2.44 seconds
--------------------------------------------------
20 / 25
current local expert:
x y t lon lat
19 400000.0 200000.0 18326.0 116.565051 85.9952
'local_data_select': 0.006 seconds
number obs: 540
setting lengthscales to: [1. 1. 1.]
'__init__': 0.025 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 1.955 seconds
'get_parameters': 0.005 seconds
parameters:
lengthscales: array([11.99995616, 7.62619823, 7.89793472])
kernel_variance: 0.010425293029739583
likelihood_variance: 0.0014469445959684995
'predict': 0.278 seconds
SAVING RESULTS TO TABLES:
run_details
preds
lengthscales
kernel_variance
likelihood_variance
total run time : 2.79 seconds
--------------------------------------------------
21 / 25
current local expert:
x y t lon lat
20 -400000.0 400000.0 18326.0 -135.0 84.933616
'local_data_select': 0.006 seconds
number obs: 450
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
**********
optimization failed!
'optimise_parameters': 2.243 seconds
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([11.99990921, 11.99986505, 2.99928517])
kernel_variance: 0.018771228552993707
likelihood_variance: 0.001740818223780367
'predict': 0.166 seconds
total run time : 2.83 seconds
--------------------------------------------------
22 / 25
current local expert:
x y t lon lat
21 -200000.0 400000.0 18326.0 -153.434949 85.9952
'local_data_select': 0.007 seconds
number obs: 543
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 2.074 seconds
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([4.97067994, 8.5397841 , 8.99999767])
kernel_variance: 0.010474033174927582
likelihood_variance: 0.001450859929606241
'predict': 0.278 seconds
total run time : 2.76 seconds
--------------------------------------------------
23 / 25
current local expert:
x y t lon lat
22 0.0 400000.0 18326.0 180.0 86.418156
'local_data_select': 0.007 seconds
number obs: 552
setting lengthscales to: [1. 1. 1.]
'__init__': 0.025 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 2.220 seconds
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([ 5.59678846, 11.99998259, 8.99999973])
kernel_variance: 0.008968445320141448
likelihood_variance: 0.001641804369442059
'predict': 0.280 seconds
total run time : 2.91 seconds
--------------------------------------------------
24 / 25
current local expert:
x y t lon lat
23 200000.0 400000.0 18326.0 153.434949 85.9952
'local_data_select': 0.006 seconds
number obs: 536
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
'optimise_parameters': 1.796 seconds
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([11.99953242, 11.99999848, 8.99999997])
kernel_variance: 0.005548404530763593
likelihood_variance: 0.0014620427250343483
'predict': 0.262 seconds
total run time : 2.47 seconds
--------------------------------------------------
25 / 25
current local expert:
x y t lon lat
24 400000.0 400000.0 18326.0 135.0 84.933616
'local_data_select': 0.006 seconds
number obs: 431
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.024 seconds
'optimise_parameters': 1.310 seconds
'get_parameters': 0.005 seconds
parameters:
lengthscales: array([11.99999699, 7.22872683, 8.10341912])
kernel_variance: 0.0057307716380633
likelihood_variance: 0.001250000052599517
'predict': 0.185 seconds
total run time : 1.90 seconds
storing any remaining tables
SAVING RESULTS TO TABLES:
run_details
preds
lengthscales
kernel_variance
likelihood_variance
'run': 72.395 seconds
results are store in hdf5
[15]:
# extract, store in dict
dfs, oi_config = get_results_from_h5file(store_path)
print(f"tables in results file: {list(dfs.keys())}")
reading in results
getting all tables
merging on expert location data
table: 'oi_config' does not have all coords_col: ['x', 'y', 't'] in columns, not merging on expert_locations
tables in results file: ['expert_locs', 'kernel_variance', 'lengthscales', 'likelihood_variance', 'oi_config', 'preds', 'run_details']
Plot Hyper Parameters
[16]:
# a template to be used for each created plot config
plot_template = {
"plot_type": "heatmap",
"x_col": "x",
"y_col": "y",
# use a northern hemisphere projection, centered at (lat,lon) = (90,0)
"subplot_kwargs": {"projection": projection},
"lat_0": lat_0,
"lon_0": lon_0,
# any additional arguments for plot_hist
"plot_kwargs": {
"scatter": True,
},
# lat/lon_col needed if scatter = True
# TODO: remove the need for this
"lat_col": "lat",
"lon_col": "lon",
}
fig = plot_hyper_parameters(dfs,
coords_col=oi_config[0]['data']['coords_col'], # ['x', 'y', 't']
row_select=None, # this could be used to select a specific date in results data
table_names=["lengthscales", "kernel_variance", "likelihood_variance"],
plot_template=plot_template,
plots_per_row=3,
suptitle="hyper params",
qvmin=0.01,
qvmax=0.99)
plt.show()
'data_select': 0.000 seconds
'load': 0.001 seconds
'data_select': 0.000 seconds
'load': 0.001 seconds
'data_select': 0.000 seconds
'load': 0.001 seconds
'data_select': 0.001 seconds
'load': 0.001 seconds
'data_select': 0.000 seconds
'load': 0.001 seconds
'data_select': 0.000 seconds
'load': 0.001 seconds
'plot_pcolormesh': 0.016 seconds
'data_select': 0.000 seconds
'load': 0.001 seconds
'plot_pcolormesh': 0.016 seconds
'data_select': 0.000 seconds
'load': 0.001 seconds
'plot_pcolormesh': 0.015 seconds
'data_select': 0.000 seconds
'load': 0.001 seconds
'plot_pcolormesh': 0.016 seconds
'data_select': 0.000 seconds
'load': 0.001 seconds
'plot_pcolormesh': 0.015 seconds
Smooth Hyper Parameters
[17]:
smooth_config = {
# get hyper parameters from the previously stored results
"result_file": store_path,
# store the smoothed hyper parameters in the same file
"output_file": store_path,
# get the hyper params from tables ending with this suffix ("" is default):
"reference_table_suffix": "",
# newly smoothed hyper parameters will be store in tables ending with table_suffix
"table_suffix": "_SMOOTHED",
# dimension names to smooth over
"xy_dims": [
"x",
"y"
],
# parameters to smooth
"params_to_smooth": [
"lengthscales",
"kernel_variance",
"likelihood_variance"
],
# length scales for the kernel smoother in each dimension
# - as well as any min/max values to apply
"smooth_config_dict": {
"lengthscales": {
"l_x": 200_000,
"l_y": 200_000
},
"likelihood_variance": {
"l_x": 200_000,
"l_y": 200_000,
"max": 0.3
},
"kernel_variance": {
"l_x": 200_000,
"l_y": 200_000,
"max": 0.1
}
},
"save_config_file": True
}
smooth_result_config_file = smooth_hyperparameters(**smooth_config)
# modify the model configuration to include "load_params"
model_load_params = model.copy()
model_load_params["load_params"] = {
"file": store_path,
"table_suffix": smooth_config["table_suffix"]
}
locexp_smooth = LocalExpertOI(expert_loc_config=local_expert,
data_config=data,
model_config=model_load_params,
pred_loc_config=pred_loc)
# run optimal interpolation (again)
# - this time don't optimise hyper parameters, but make predictions
# - store results in new tables ending with '_SMOOTHED'
locexp_smooth.run(store_path=store_path,
optimise=False,
predict=True,
table_suffix=smooth_config['table_suffix'],
check_config_compatible=False)
found model_name: GPflowGPRModel
setting lengthscales to: [1.]
'__init__': 0.025 seconds
reading in results
selecting only tables: ['lengthscales', 'kernel_variance', 'likelihood_variance']
'data_select': 0.000 seconds
'load': 0.001 seconds
expert_locations data will not be merged on results data
adding smoothed table: lengthscales_SMOOTHED
adding smoothed table: kernel_variance_SMOOTHED
adding smoothed table: likelihood_variance_SMOOTHED
writing (smoothed) hyper parameters to:
/home/runner/work/GPSat/GPSat/results/inline_example.h5
table_suffix:_SMOOTHED
writing: lengthscales_SMOOTHED to table
writing: kernel_variance_SMOOTHED to table
writing: likelihood_variance_SMOOTHED to table
writing config (to use to make predictions with smoothed values) to:
/home/runner/work/GPSat/GPSat/results/inline_example_SMOOTHED.json
'data_select': 0.000 seconds
'load': 0.000 seconds
in json_serializable - key: 'data_source' has value DataFrame/Series, but is too long: 22522 > 100
storing as str
in json_serializable - key: 'df' has value DataFrame/Series, but is too long: 40000 > 100
storing as str
---------
storing expert locations in 'expert_locs' table
exception occurred: 'No object named expert_locs_SMOOTHED in the file'
will now close object
---------
dropping expert locations that already exists in 'run_details' table
exception occurred: 'No object named run_details_SMOOTHED in the file'
will now close object
--------------------------------------------------
1 / 25
current local expert:
x y t lon lat
0 -400000.0 -400000.0 18326.0 -45.0 84.933616
'data_select': 0.000 seconds
'load': 0.000 seconds
'local_data_select': 0.006 seconds
number obs: 402
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'get_parameters': 0.005 seconds
'_read_params_from_file': 0.066 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([5.73482601, 6.15213143, 7.51410806])
kernel_variance: 0.018189635819469134
likelihood_variance: 0.005625
'predict': 0.147 seconds
total run time : 0.62 seconds
--------------------------------------------------
2 / 25
current local expert:
x y t lon lat
1 -200000.0 -400000.0 18326.0 -26.565051 85.9952
'local_data_select': 0.006 seconds
number obs: 538
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.063 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.022 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([5.72410962, 6.29331535, 8.21639456])
kernel_variance: 0.016447043701965577
likelihood_variance: 0.005625
'predict': 0.252 seconds
total run time : 0.73 seconds
--------------------------------------------------
3 / 25
current local expert:
x y t lon lat
2 0.0 -400000.0 18326.0 0.0 86.418156
'local_data_select': 0.006 seconds
number obs: 519
setting lengthscales to: [1. 1. 1.]
'__init__': 0.025 seconds
'_read_params_from_file': 0.063 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([6.22036061, 6.31743778, 8.70618362])
kernel_variance: 0.013843509876526537
likelihood_variance: 0.005625
'predict': 0.249 seconds
total run time : 0.72 seconds
--------------------------------------------------
4 / 25
current local expert:
x y t lon lat
3 200000.0 -400000.0 18326.0 26.565051 85.9952
'local_data_select': 0.006 seconds
number obs: 525
setting lengthscales to: [1. 1. 1.]
'__init__': 0.025 seconds
'_read_params_from_file': 0.063 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([7.24807951, 5.96802821, 8.82122936])
kernel_variance: 0.013041798224101255
likelihood_variance: 0.005625
'predict': 0.250 seconds
total run time : 0.73 seconds
--------------------------------------------------
5 / 25
current local expert:
x y t lon lat
4 400000.0 -400000.0 18326.0 45.0 84.933616
'local_data_select': 0.006 seconds
number obs: 409
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.063 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([8.10321495, 5.64199366, 8.84037421])
kernel_variance: 0.013224941185319604
likelihood_variance: 0.005625
'predict': 0.158 seconds
total run time : 0.64 seconds
--------------------------------------------------
6 / 25
current local expert:
x y t lon lat
5 -400000.0 -200000.0 18326.0 -63.434949 85.9952
'local_data_select': 0.006 seconds
number obs: 566
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.063 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([6.95913876, 8.44033542, 6.19299764])
kernel_variance: 0.020667054178272058
likelihood_variance: 0.005625
'predict': 0.272 seconds
total run time : 0.75 seconds
--------------------------------------------------
7 / 25
current local expert:
x y t lon lat
6 -200000.0 -200000.0 18326.0 -45.0 87.467477
'local_data_select': 0.006 seconds
number obs: 548
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.062 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([7.2633832 , 8.1029856 , 7.33817359])
kernel_variance: 0.018748865242621378
likelihood_variance: 0.005625
'predict': 0.309 seconds
total run time : 0.79 seconds
--------------------------------------------------
8 / 25
current local expert:
x y t lon lat
7 0.0 -200000.0 18326.0 0.0 88.209314
'local_data_select': 0.006 seconds
number obs: 501
setting lengthscales to: [1. 1. 1.]
'__init__': 0.025 seconds
'_read_params_from_file': 0.063 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.022 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([7.71919297, 7.36708431, 8.34013836])
kernel_variance: 0.014902528637110597
likelihood_variance: 0.005625
'predict': 0.273 seconds
total run time : 0.75 seconds
--------------------------------------------------
9 / 25
current local expert:
x y t lon lat
8 200000.0 -200000.0 18326.0 45.0 87.467477
'local_data_select': 0.006 seconds
number obs: 525
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.063 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([8.49048042, 6.49304497, 8.70552745])
kernel_variance: 0.012851982949023113
likelihood_variance: 0.005625
'predict': 0.288 seconds
total run time : 0.76 seconds
--------------------------------------------------
10 / 25
current local expert:
x y t lon lat
9 400000.0 -200000.0 18326.0 63.434949 85.9952
'local_data_select': 0.006 seconds
number obs: 569
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.062 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.022 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([9.26019183, 6.0745956 , 8.76470889])
kernel_variance: 0.012491209161613902
likelihood_variance: 0.005625
'predict': 0.280 seconds
SAVING RESULTS TO TABLES:
run_details
preds
total run time : 0.82 seconds
--------------------------------------------------
11 / 25
current local expert:
x y t lon lat
10 -400000.0 0.0 18326.0 -90.0 86.418156
'local_data_select': 0.006 seconds
number obs: 591
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.062 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([ 8.27405142, 10.22251749, 5.01750181])
kernel_variance: 0.021152314546381622
likelihood_variance: 0.005625
'predict': 0.296 seconds
total run time : 0.78 seconds
--------------------------------------------------
12 / 25
current local expert:
x y t lon lat
11 -200000.0 0.0 18326.0 -90.0 88.209314
'local_data_select': 0.006 seconds
number obs: 516
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.062 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([8.84813885, 9.58116694, 6.38660579])
kernel_variance: 0.019122731228315436
likelihood_variance: 0.005625
'predict': 0.287 seconds
total run time : 0.76 seconds
--------------------------------------------------
13 / 25
current local expert:
x y t lon lat
12 0.0 0.0 18326.0 0.0 90.0
'local_data_select': 0.006 seconds
number obs: 522
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.062 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.022 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([9.13897742, 8.70000241, 7.91471655])
kernel_variance: 0.01466006136518658
likelihood_variance: 0.005625
'predict': 0.290 seconds
total run time : 0.77 seconds
--------------------------------------------------
14 / 25
current local expert:
x y t lon lat
13 200000.0 0.0 18326.0 90.0 88.209314
'local_data_select': 0.006 seconds
number obs: 501
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.062 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.022 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([9.5138172 , 7.73038781, 8.61930299])
kernel_variance: 0.011779774154452418
likelihood_variance: 0.005625
'predict': 0.274 seconds
total run time : 0.74 seconds
--------------------------------------------------
15 / 25
current local expert:
x y t lon lat
14 400000.0 0.0 18326.0 90.0 86.418156
'local_data_select': 0.006 seconds
number obs: 564
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.063 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.022 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([10.07619809, 7.07199748, 8.69076398])
kernel_variance: 0.010949949168978873
likelihood_variance: 0.005625
'predict': 0.275 seconds
total run time : 0.75 seconds
--------------------------------------------------
16 / 25
current local expert:
x y t lon lat
15 -400000.0 200000.0 18326.0 -116.565051 85.9952
'local_data_select': 0.006 seconds
number obs: 556
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.063 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.022 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([ 8.87492853, 10.96939231, 4.67469361])
kernel_variance: 0.019387299428597616
likelihood_variance: 0.005625
'predict': 0.270 seconds
total run time : 0.75 seconds
--------------------------------------------------
17 / 25
current local expert:
x y t lon lat
16 -200000.0 200000.0 18326.0 -135.0 87.467477
'local_data_select': 0.006 seconds
number obs: 556
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.061 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.022 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([ 9.0243556 , 10.32438304, 6.15638749])
kernel_variance: 0.016965127896462175
likelihood_variance: 0.005625
'predict': 0.314 seconds
total run time : 0.78 seconds
--------------------------------------------------
18 / 25
current local expert:
x y t lon lat
17 0.0 200000.0 18326.0 180.0 88.209314
'local_data_select': 0.006 seconds
number obs: 513
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.063 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([9.09969537, 9.77991414, 7.83632577])
kernel_variance: 0.012666506508756673
likelihood_variance: 0.005625
'predict': 0.279 seconds
total run time : 0.76 seconds
--------------------------------------------------
19 / 25
current local expert:
x y t lon lat
18 200000.0 200000.0 18326.0 135.0 87.467477
'local_data_select': 0.006 seconds
number obs: 532
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.062 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([9.67692024, 9.15479749, 8.58026479])
kernel_variance: 0.009829325080917857
likelihood_variance: 0.005625
'predict': 0.296 seconds
total run time : 0.77 seconds
--------------------------------------------------
20 / 25
current local expert:
x y t lon lat
19 400000.0 200000.0 18326.0 116.565051 85.9952
'local_data_select': 0.006 seconds
number obs: 540
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.061 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.022 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([10.49409551, 8.30007457, 8.55474342])
kernel_variance: 0.009064997133667227
likelihood_variance: 0.005625
'predict': 0.258 seconds
SAVING RESULTS TO TABLES:
run_details
preds
total run time : 0.82 seconds
--------------------------------------------------
21 / 25
current local expert:
x y t lon lat
20 -400000.0 400000.0 18326.0 -135.0 84.933616
'local_data_select': 0.006 seconds
number obs: 450
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.064 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.023 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([ 8.99881649, 11.05755859, 4.95603326])
kernel_variance: 0.017317448141288112
likelihood_variance: 0.005625
'predict': 0.171 seconds
total run time : 0.65 seconds
--------------------------------------------------
22 / 25
current local expert:
x y t lon lat
21 -200000.0 400000.0 18326.0 -153.434949 85.9952
'local_data_select': 0.006 seconds
number obs: 543
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.063 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.022 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([ 8.32685343, 10.54930775, 6.60105582])
kernel_variance: 0.014479552888272865
likelihood_variance: 0.005625
'predict': 0.263 seconds
total run time : 0.74 seconds
--------------------------------------------------
23 / 25
current local expert:
x y t lon lat
22 0.0 400000.0 18326.0 180.0 86.418156
'local_data_select': 0.006 seconds
number obs: 552
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.062 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.023 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([ 8.40517259, 10.40807104, 8.11292482])
kernel_variance: 0.01066958298913049
likelihood_variance: 0.005625
'predict': 0.267 seconds
total run time : 0.73 seconds
--------------------------------------------------
24 / 25
current local expert:
x y t lon lat
23 200000.0 400000.0 18326.0 153.434949 85.9952
'local_data_select': 0.006 seconds
number obs: 536
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.063 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.007 seconds
'set_likelihood_variance_constraints': 0.022 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([ 9.67700925, 10.02468704, 8.61816687])
kernel_variance: 0.008194074425186689
likelihood_variance: 0.005625
'predict': 0.254 seconds
total run time : 0.72 seconds
--------------------------------------------------
25 / 25
current local expert:
x y t lon lat
24 400000.0 400000.0 18326.0 135.0 84.933616
'local_data_select': 0.006 seconds
number obs: 431
setting lengthscales to: [1. 1. 1.]
'__init__': 0.024 seconds
'_read_params_from_file': 0.062 seconds
'set_parameters': 0.007 seconds
'set_lengthscales_constraints': 0.006 seconds
'set_likelihood_variance_constraints': 0.022 seconds
*** not optimising parameters
'get_parameters': 0.004 seconds
parameters:
lengthscales: array([10.89511901, 8.99245346, 8.47346642])
kernel_variance: 0.007517505054520759
likelihood_variance: 0.005625
'predict': 0.167 seconds
total run time : 0.64 seconds
storing any remaining tables
SAVING RESULTS TO TABLES:
run_details
preds
'run': 18.619 seconds
Plot Smoothed Hyper Parameters
[18]:
# extract, store in dict
dfs, oi_config = get_results_from_h5file(store_path)
fig = plot_hyper_parameters(dfs,
coords_col=oi_config[0]['data']['coords_col'], # ['x', 'y', 't']
row_select=None,
table_names=["lengthscales", "kernel_variance", "likelihood_variance"],
table_suffix=smooth_config["table_suffix"],
plot_template=plot_template,
plots_per_row=3,
suptitle="smoothed hyper params",
qvmin=0.01,
qvmax=0.99)
plt.tight_layout()
plt.show()
reading in results
getting all tables
merging on expert location data
table: 'oi_config' does not have all coords_col: ['x', 'y', 't'] in columns, not merging on expert_locations
table: 'oi_config_SMOOTHED' does not have all coords_col: ['x', 'y', 't'] in columns, not merging on expert_locations
'data_select': 0.000 seconds
'load': 0.001 seconds
'data_select': 0.000 seconds
'load': 0.000 seconds
'data_select': 0.000 seconds
'load': 0.000 seconds
'data_select': 0.000 seconds
'load': 0.000 seconds
'data_select': 0.000 seconds
'load': 0.000 seconds
'data_select': 0.000 seconds
'load': 0.001 seconds
'plot_pcolormesh': 0.015 seconds
'data_select': 0.000 seconds
'load': 0.001 seconds
'plot_pcolormesh': 0.015 seconds
'data_select': 0.000 seconds
'load': 0.001 seconds
'plot_pcolormesh': 0.015 seconds
'data_select': 0.000 seconds
'load': 0.001 seconds
'plot_pcolormesh': 0.015 seconds
'data_select': 0.000 seconds
'load': 0.001 seconds
'plot_pcolormesh': 0.014 seconds
get weighted combinations predictions and plot
[19]:
plt_data = dfs["preds" + smooth_config["table_suffix"]]
weighted_values_kwargs = {
"ref_col": ["pred_loc_x", "pred_loc_y", "pred_loc_t"],
"dist_to_col": ["x", "y", "t"],
"val_cols": ["f*", "f*_var"],
"weight_function": "gaussian",
"lengthscale": inference_radius/2
}
plt_data = get_weighted_values(df=plt_data, **weighted_values_kwargs)
plt_data['lon'], plt_data['lat'] = EASE2toWGS84(plt_data['pred_loc_x'], plt_data['pred_loc_y'])
fig = plt.figure(figsize=(12, 12))
ax = fig.add_subplot(1, 1, 1, projection=get_projection(projection))
plot_pcolormesh_from_results_data(ax=ax,
dfs={"preds": plt_data},
table='preds',
val_col="f*",
x_col='pred_loc_x',
y_col='pred_loc_y',
fig=fig)
plt.tight_layout()
plt.show()
'data_select': 0.001 seconds
'load': 0.001 seconds
'plot_pcolormesh': 0.034 seconds
