From c50f4d226a5c3dcabe9f7594ae659c31ba5cfb36 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 3 Jun 2026 10:50:32 -0700 Subject: [PATCH 1/2] Add test for stoqs url creation. Problem not here. Data hadn't loaded into stoqs yet. --- .vscode/launch.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 0bf9dec..050ebee 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -503,12 +503,14 @@ // Test a month with fewer plots to make //"args": ["--auv_name", "daphne", "--start", "20260101", "--end", "20260131", "-v", "2", "--clobber"] // Test pyxis and cbit_amphoursused plot - "args": ["--auv_name", "pyxis", "--start", "20260526", "--end", "20260531", "-v", "1", "--clobber"] + //"args": ["--auv_name", "pyxis", "--start", "20260526", "--end", "20260531", "-v", "1", "--clobber"] // Test --current_month //"args": ["--current_month", "-v", "1"] // Test reading downwelling_photosynthetic // _photon_flux_in_sea_water from the root group of the netCDF file instead of the instrument group, which was the case for some early daphne log files //"args": ["--auv_name", "daphne", "--start", "20260501", "--end", "20260531", "-v", "1", "--clobber"] + // Test stoqs url creation with valid start and end times - problem was that data hadn't loaded into stoqs yet. + "args": ["--auv_name", "makai", "--start", "20260601", "--end", "20260604", "-v", "1", "--clobber"] }, { "name": "lrauv_deployment_plots", From 1366aab13fc5d0a28b5f48f3f5141357859360b1 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 3 Jun 2026 17:04:36 -0700 Subject: [PATCH 2/2] Cope with missing/incomplete data so that we can at least get the cbit plot. --- .vscode/launch.json | 3 +- src/data/create_products.py | 63 +++++++++++++++++++++++++------------ 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 050ebee..a17bf62 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -510,7 +510,8 @@ // _photon_flux_in_sea_water from the root group of the netCDF file instead of the instrument group, which was the case for some early daphne log files //"args": ["--auv_name", "daphne", "--start", "20260501", "--end", "20260531", "-v", "1", "--clobber"] // Test stoqs url creation with valid start and end times - problem was that data hadn't loaded into stoqs yet. - "args": ["--auv_name", "makai", "--start", "20260601", "--end", "20260604", "-v", "1", "--clobber"] + //"args": ["--auv_name", "makai", "--start", "20260601", "--end", "20260604", "-v", "1", "--clobber"] + "args": ["--auv_name", "pyxis", "--current_month", "-v", "1"] }, { "name": "lrauv_deployment_plots", diff --git a/src/data/create_products.py b/src/data/create_products.py index a0a8659..75ad39e 100755 --- a/src/data/create_products.py +++ b/src/data/create_products.py @@ -1070,32 +1070,41 @@ def _grid_dims(self, plot_vars: list[str] | None = None) -> tuple: attrs=distnav.attrs, ) - # Horizontal gridded to 3x the number of profiles + # Horizontal gridded to 3x the number of profiles (or len of distnav if unavailable) + num_profiles = ( + int(np.nanmax(self.ds["profile_number"].to_numpy())) + if "profile_number" in self.ds + else len(distnav) + ) idist = np.linspace( distnav.to_numpy()[0], distnav.to_numpy()[-1], - int(3 * np.nanmax(self.ds["profile_number"].to_numpy())), + 3 * num_profiles, ) # Vertical gridded to .5 m, rounded down to nearest 10m (minimum 10m) # Use only depths where at least one sensor variable has valid data to # exclude bogus depth values recorded when no valid sensor data was logged # (e.g. from memory corruption events) - depth_values = self.ds.cf["depth"].to_numpy() - time_dim = self.ds.cf["depth"].dims[0] - nav_vars = {"depth", "latitude", "longitude", "profile_number"} - has_valid_sensor_data = np.zeros(len(depth_values), dtype=bool) - vars_to_check = [ - v for v in (plot_vars or self.ds.data_vars) if v in self.ds and "pitch" not in v - ] - for var in vars_to_check: - if var not in nav_vars and time_dim in self.ds[var].dims and self.ds[var].ndim == 1: - has_valid_sensor_data |= ~np.isnan(self.ds[var].to_numpy()) - depths_with_data = depth_values[has_valid_sensor_data] - if len(depths_with_data) > 0 and not np.all(np.isnan(depths_with_data)): - max_depth = max(np.floor(np.nanmax(depths_with_data) / 10) * 10, 10) + if "depth" not in self.ds.cf: + self.logger.debug("No depth variable in dataset, using minimal iz") + iz = np.arange(0, 10.5, 0.5) else: - max_depth = max(np.floor(np.nanmax(depth_values) / 10) * 10, 10) - iz = np.arange(0, max_depth + 0.5, 0.5) # include max_depth so axis reaches it + depth_values = self.ds.cf["depth"].to_numpy() + time_dim = self.ds.cf["depth"].dims[0] + nav_vars = {"depth", "latitude", "longitude", "profile_number"} + has_valid_sensor_data = np.zeros(len(depth_values), dtype=bool) + vars_to_check = [ + v for v in (plot_vars or self.ds.data_vars) if v in self.ds and "pitch" not in v + ] + for var in vars_to_check: + if var not in nav_vars and time_dim in self.ds[var].dims and self.ds[var].ndim == 1: + has_valid_sensor_data |= ~np.isnan(self.ds[var].to_numpy()) + depths_with_data = depth_values[has_valid_sensor_data] + if len(depths_with_data) > 0 and not np.all(np.isnan(depths_with_data)): + max_depth = max(np.floor(np.nanmax(depths_with_data) / 10) * 10, 10) + else: + max_depth = max(np.floor(np.nanmax(depth_values) / 10) * 10, 10) + iz = np.arange(0, max_depth + 0.5, 0.5) # include max_depth so axis reaches it return idist, iz, distnav @@ -1166,6 +1175,9 @@ def _profile_bottoms( """Return array of distance and depth points defining the bottom of the profiles where there is no data""" + if "depth" not in self.ds.cf: + self.logger.debug("No depth variable in dataset, skipping profile bottoms") + return None # Create a DataArray of depths indexed by distance depth_dist = xr.DataArray( self.ds.cf["depth"].to_numpy(), @@ -1464,7 +1476,13 @@ def _section_overlays(self, distnav: xr.DataArray) -> tuple: return profile_bottoms, bottom_depths def _cumulative_profile_numbers(self) -> np.ndarray: - """Return profile_number array that increments monotonically across log files.""" + """Return profile_number array that increments monotonically across log files. + + Falls back to a simple index array when profile_number is absent. + """ + if "profile_number" not in self.ds: + self.logger.debug("profile_number not in dataset, using index array for track color") + return np.arange(len(self.ds["time"]), dtype=float) profile_numbers = self.ds["profile_number"].to_numpy() result = profile_numbers.copy().astype(float) offset = 0 @@ -2066,14 +2084,19 @@ def _plot_var_scatter( # noqa: C901, PLR0912, PLR0913, PLR0915 # filtered in _grid_dims (e.g. sparse GPS in realtime SBD data). # Align depth and var arrays to distnav's time subset. n_ds = len(self.ds.cf["time"]) + has_depth = "depth" in self.ds.cf if len(distnav) != n_ds: ds_time = self.ds.cf["time"].to_numpy() nav_idx = np.searchsorted(ds_time, distnav.coords["time"].to_numpy()) nav_idx = nav_idx[nav_idx < n_ds] - depth_for_scatter = self.ds.cf["depth"].to_numpy()[nav_idx] + depth_for_scatter = ( + self.ds.cf["depth"].to_numpy()[nav_idx] if has_depth else np.zeros(len(nav_idx)) + ) var_for_scatter = var_to_plot[nav_idx] else: - depth_for_scatter = self.ds.cf["depth"].to_numpy() + depth_for_scatter = ( + self.ds.cf["depth"].to_numpy() if has_depth else np.zeros(len(var_to_plot)) + ) var_for_scatter = var_to_plot # Create scatter plot with actual data points