Use all observations

Author: nefrathenrici

experiments/calibration/Project.toml

@@ -11,6 +11,7 @@ ClimaCoreMakie = "908f55d8-4145-4867-9c14-5dad1a479e4d"
 ClimaCoupler = "4ade58fe-a8da-486c-bd89-46df092ec0c7"
 ClimaDiagnostics = "1ecacbb8-0713-4841-9a07-eb5aa8a2d53f"
 ClimaLand = "08f4d4ce-cf43-44bb-ad95-9d2d5f413532"
+ClimaParams = "5c42b081-d73a-476f-9059-fd94b934656c"
 ClimaTimeSteppers = "595c0a79-7f3d-439a-bc5a-b232dc3bde79"
 ClimaUtilities = "b3f4f4ca-9299-4f7f-bd9b-81e1242a7513"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
@@ -31,7 +32,7 @@ YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
 
 [compat]
 ClimaCalibrate = "0.0.11"
-ClimaLand = "=0.15.9"
-ClimaTimeSteppers = "0.8.2"
 ClimaCore = "0.14.26"
+ClimaLand = "0.15.11"
+ClimaTimeSteppers = "0.8.2"
 EnsembleKalmanProcesses = "2.1.2"

experiments/calibration/coarse_amip/model_config.yml

@@ -46,5 +46,7 @@ extra_atmos_diagnostics:
       - clw
       - cli
       - pfull
+      - clw
+      - cli
     period: 1months
     writer: nc

experiments/calibration/coarse_amip/model_interface.jl

@@ -29,15 +29,14 @@ function ClimaCalibrate.forward_model(iter, member)
     # Set member output directory
     member_output_dir = ClimaCalibrate.path_to_ensemble_member(output_dir_root, iter, member)
     config_dict["coupler_output_dir"] = member_output_dir
-    sim = try
-        setup_and_run(config_dict)
-    catch e
-        @error "Forward model error" exception = e
-        bt = catch_backtrace()
-        println("Stacktrace:")
-        display(stacktrace(bt))
-        nothing
-    end
+    sim = setup_and_run(config_dict)
+    # catch e
+    #     @error "Forward model error" exception = e
+    #     bt = catch_backtrace()
+    #     println("Stacktrace:")
+    #     display(stacktrace(bt))
+    #     nothing
+    # end
     @info "Completed member $member"
     return sim
 end

experiments/calibration/coarse_amip/observation_map.jl

@@ -17,10 +17,7 @@ function ClimaCalibrate.observation_map(iteration)
             G_ensemble[:, m] .= process_member_data(SimDir(simdir_path))
 
         catch e
-            @error "Error processing member $m, filling observation map entry with NaNs" exception = e
-            bt = catch_backtrace()
-            println("Stacktrace:")
-            display(stacktrace(bt))
+            @error "Error processing member $m, filling observation map entry with NaNs" exception = e 
             G_ensemble[:, m] .= NaN
         end
     end
@@ -46,8 +43,8 @@ function process_member_data(simdir::SimDir)
     hur = process_outputvar(simdir, "hur")
     hus = process_outputvar(simdir, "hus")
 
-    ql = process_outputvar(simdir, "ql")
-    qi = process_outputvar(simdir, "qi")
+    clw = process_outputvar(simdir, "clw")
+    cli = process_outputvar(simdir, "cli")
 
     # Map over each year
     year_observations = map(1:4:length(rsut)) do year_start
@@ -66,10 +63,10 @@ function process_member_data(simdir::SimDir)
         ta_yr = downsample_and_vectorize(ta[yr_ind])
         hur_yr = downsample_and_vectorize(hur[yr_ind])
         hus_yr = downsample_and_vectorize(hus[yr_ind])
-        ql_yr = downsample_and_vectorize(ql[yr_ind])
-        qi_yr = downsample_and_vectorize(qi[yr_ind])
+        clw_yr = downsample_and_vectorize(clw[yr_ind])
+        cli_yr = downsample_and_vectorize(cli[yr_ind])
 
-        vcat(net_rad_yr, rsut_yr, rlut_yr, cre_yr, pr_yr, shf_yr, ts_yr, ta_yr, hur_yr, hus_yr, ql_yr, qi_yr)
+        vcat(net_rad_yr, rsut_yr, rlut_yr, cre_yr, pr_yr, shf_yr, ts_yr, ta_yr, hur_yr, hus_yr, clw_yr, cli_yr)
     end
     return vcat(year_observations...)
 end
@@ -93,18 +90,20 @@ end
 function preprocess_monthly_averages(simdir, name)
     monthly_avgs = get_monthly_averages(simdir, name)
     # Interpolate to pressure coordinates to match observations
+    pressure = get_monthly_averages(simdir, "pfull")
     if has_altitude(monthly_avgs)
-        pressure = get_monthly_averages(simdir, "pfull")
         monthly_avgs = ClimaAnalysis.Atmos.to_pressure_coordinates(monthly_avgs, pressure)
         monthly_avgs = limit_pressure_dim_to_era5_range(monthly_avgs)
     end
+    monthly_avgs.attributes["start_date"] = pressure.attributes["start_date"]
     # Line up dates for monthly averages
     monthly_avgs = ClimaAnalysis.shift_to_start_of_previous_month(monthly_avgs)
     # Remove spinup time
     monthly_avgs = window(monthly_avgs, "time"; left = spinup_time)
     global_mean = monthly_avgs |> average_lat |> average_lon |> average_time
+    FT = monthly_avgs.data |> eltype
     # Replace NaNs with global mean
-    monthly_avgs = ClimaAnalysis.replace(monthly_avgs, NaN => global_mean)
+    monthly_avgs = ClimaAnalysis.replace(monthly_avgs, NaN => FT(mean(global_mean.data)))
     return monthly_avgs
 end
 

experiments/calibration/coarse_amip/run_calibration.jl

@@ -10,6 +10,7 @@ import EnsembleKalmanProcesses as EKP
 
 include(joinpath(pkgdir(ClimaCoupler), "experiments/calibration/coarse_amip/observation_map.jl"))
 
+# addprocs(CAL.SlurmManager(30); partition = "a3", gpus_per_task = 1, cpus_per_task = 4, time = "08:00:00")
 addprocs(CAL.SlurmManager())
 
 # Make variables and the forward model available on the worker sessions