NetCDF 4.9.1-rc1
nc4hdf.c
Go to the documentation of this file.
1/* Copyright 2018, University Corporation for Atmospheric
2 * Research. See the COPYRIGHT file for copying and redistribution
3 * conditions. */
17#include "config.h"
18#include "netcdf.h"
19#include "nc4internal.h"
20#include "ncdispatch.h"
21#include "hdf5internal.h"
22#include "hdf5err.h" /* For BAIL2 */
23#include "hdf5debug.h"
24#include <math.h>
25
26#ifdef HAVE_INTTYPES_H
27#define __STDC_FORMAT_MACROS
28#include <inttypes.h>
29#endif
30
31#define NC_HDF5_MAX_NAME 1024
41static int
42flag_atts_dirty(NCindex *attlist) {
43
44 NC_ATT_INFO_T *att = NULL;
45 int i;
46
47 if(attlist == NULL) {
48 return NC_NOERR;
49 }
50
51 for(i=0;i<ncindexsize(attlist);i++) {
52 att = (NC_ATT_INFO_T*)ncindexith(attlist,i);
53 if(att == NULL) continue;
54 att->dirty = NC_TRUE;
55 }
56
57 return NC_NOERR;
58}
59
76int
77rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid)
78{
79 NC_VAR_INFO_T *var;
80 NC_GRP_INFO_T *child_grp;
81 int d, i;
82 int retval;
83
84 assert(grp && grp->hdr.name && dimid >= 0 && dimscaleid >= 0);
85 LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
86
87 /* If there are any child groups, attach dimscale there, if needed. */
88 for (i = 0; i < ncindexsize(grp->children); i++)
89 {
90 child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children, i);
91 assert(child_grp);
92 if ((retval = rec_reattach_scales(child_grp, dimid, dimscaleid)))
93 return retval;
94 }
95
96 /* Find any vars that use this dimension id. */
97 for (i = 0; i < ncindexsize(grp->vars); i++)
98 {
99 NC_HDF5_VAR_INFO_T *hdf5_var;
100
101 var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i);
102 assert(var && var->format_var_info);
103
104 hdf5_var = (NC_HDF5_VAR_INFO_T*)var->format_var_info;
105 assert(hdf5_var != NULL);
106 for (d = 0; d < var->ndims; d++)
107 {
108 if (var->dimids[d] == dimid && !hdf5_var->dimscale)
109 {
110 LOG((2, "%s: attaching scale for dimid %d to var %s",
111 __func__, var->dimids[d], var->hdr.name));
112 if (var->created)
113 {
114 if (H5DSattach_scale(hdf5_var->hdf_datasetid,
115 dimscaleid, d) < 0)
116 return NC_EDIMSCALE;
117 hdf5_var->dimscale_attached[d] = NC_TRUE;
118 }
119 }
120 }
121 }
122 return NC_NOERR;
123}
124
141int
142rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid)
143{
144 NC_VAR_INFO_T *var;
145 NC_GRP_INFO_T *child_grp;
146 int d, i;
147 int retval;
148
149 assert(grp && grp->hdr.name && dimid >= 0 && dimscaleid >= 0);
150 LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
151
152 /* If there are any child groups, detach dimscale there, if needed. */
153 for(i=0;i<ncindexsize(grp->children);i++) {
154 child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children,i);
155 if(child_grp == NULL) continue;
156 if ((retval = rec_detach_scales(child_grp, dimid, dimscaleid)))
157 return retval;
158 }
159
160 /* Find any vars that use this dimension id. */
161 for (i = 0; i < ncindexsize(grp->vars); i++)
162 {
163 NC_HDF5_VAR_INFO_T *hdf5_var;
164 var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
165 assert(var && var->format_var_info);
166 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
167
168 for (d = 0; d < var->ndims; d++)
169 {
170 if (var->dimids[d] == dimid && !hdf5_var->dimscale)
171 {
172 LOG((2, "%s: detaching scale for dimid %d to var %s",
173 __func__, var->dimids[d], var->hdr.name));
174 if (var->created)
175 {
176 if (hdf5_var->dimscale_attached && hdf5_var->dimscale_attached[d])
177 {
178 if (H5DSdetach_scale(hdf5_var->hdf_datasetid,
179 dimscaleid, d) < 0)
180 return NC_EDIMSCALE;
181 hdf5_var->dimscale_attached[d] = NC_FALSE;
182 }
183 }
184 }
185 }
186 }
187 return NC_NOERR;
188}
189
201int
202nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset)
203{
204 NC_VAR_INFO_T *var;
205 NC_HDF5_VAR_INFO_T *hdf5_var;
206
207 assert(grp && grp->format_grp_info && dataset);
208
209 /* Find the requested varid. */
210 if (!(var = (NC_VAR_INFO_T *)ncindexith(grp->vars, varid)))
211 return NC_ENOTVAR;
212 assert(var && var->hdr.id == varid && var->format_var_info);
213 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
214
215 /* Open this dataset if necessary. */
216 if (!hdf5_var->hdf_datasetid)
217 {
218 NC_HDF5_GRP_INFO_T *hdf5_grp;
219 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
220
221 if ((hdf5_var->hdf_datasetid = H5Dopen2(hdf5_grp->hdf_grpid,
222 var->hdr.name, H5P_DEFAULT)) < 0)
223 return NC_ENOTVAR;
224 }
225
226 *dataset = hdf5_var->hdf_datasetid;
227
228 return NC_NOERR;
229}
230
248int
249nc4_get_hdf_typeid(NC_FILE_INFO_T *h5, nc_type xtype,
250 hid_t *hdf_typeid, int endianness)
251{
252 NC_TYPE_INFO_T *type;
253 hid_t typeid = 0;
254 int retval = NC_NOERR;
255
256 assert(hdf_typeid && h5);
257
258 *hdf_typeid = -1;
259
260 /* Determine an appropriate HDF5 datatype */
261 if (xtype == NC_NAT)
262 return NC_EBADTYPE;
263 else if (xtype == NC_CHAR || xtype == NC_STRING)
264 {
265 /* NC_CHAR & NC_STRING types create a new HDF5 datatype */
266 if (xtype == NC_CHAR)
267 {
268 if ((typeid = H5Tcopy(H5T_C_S1)) < 0)
269 return NC_EHDFERR;
270 if (H5Tset_strpad(typeid, H5T_STR_NULLTERM) < 0)
271 BAIL(NC_EVARMETA);
272 if(H5Tset_cset(typeid, H5T_CSET_ASCII) < 0)
273 BAIL(NC_EVARMETA);
274
275 /* Take ownership of the newly created HDF5 datatype */
276 *hdf_typeid = typeid;
277 typeid = 0;
278 }
279 else
280 {
281 if ((typeid = H5Tcopy(H5T_C_S1)) < 0)
282 return NC_EHDFERR;
283 if (H5Tset_size(typeid, H5T_VARIABLE) < 0)
284 BAIL(NC_EVARMETA);
285 if(H5Tset_cset(typeid, H5T_CSET_UTF8) < 0)
286 BAIL(NC_EVARMETA);
287
288 /* Take ownership of the newly created HDF5 datatype */
289 *hdf_typeid = typeid;
290 typeid = 0;
291 }
292 }
293 else
294 {
295 /* All other types use an existing HDF5 datatype */
296 switch (xtype)
297 {
298 case NC_BYTE: /* signed 1 byte integer */
299 if (endianness == NC_ENDIAN_LITTLE)
300 typeid = H5T_STD_I8LE;
301 else if (endianness == NC_ENDIAN_BIG)
302 typeid = H5T_STD_I8BE;
303 else
304 typeid = H5T_NATIVE_SCHAR;
305 break;
306
307 case NC_SHORT: /* signed 2 byte integer */
308 if (endianness == NC_ENDIAN_LITTLE)
309 typeid = H5T_STD_I16LE;
310 else if (endianness == NC_ENDIAN_BIG)
311 typeid = H5T_STD_I16BE;
312 else
313 typeid = H5T_NATIVE_SHORT;
314 break;
315
316 case NC_INT:
317 if (endianness == NC_ENDIAN_LITTLE)
318 typeid = H5T_STD_I32LE;
319 else if (endianness == NC_ENDIAN_BIG)
320 typeid = H5T_STD_I32BE;
321 else
322 typeid = H5T_NATIVE_INT;
323 break;
324
325 case NC_UBYTE:
326 if (endianness == NC_ENDIAN_LITTLE)
327 typeid = H5T_STD_U8LE;
328 else if (endianness == NC_ENDIAN_BIG)
329 typeid = H5T_STD_U8BE;
330 else
331 typeid = H5T_NATIVE_UCHAR;
332 break;
333
334 case NC_USHORT:
335 if (endianness == NC_ENDIAN_LITTLE)
336 typeid = H5T_STD_U16LE;
337 else if (endianness == NC_ENDIAN_BIG)
338 typeid = H5T_STD_U16BE;
339 else
340 typeid = H5T_NATIVE_USHORT;
341 break;
342
343 case NC_UINT:
344 if (endianness == NC_ENDIAN_LITTLE)
345 typeid = H5T_STD_U32LE;
346 else if (endianness == NC_ENDIAN_BIG)
347 typeid = H5T_STD_U32BE;
348 else
349 typeid = H5T_NATIVE_UINT;
350 break;
351
352 case NC_INT64:
353 if (endianness == NC_ENDIAN_LITTLE)
354 typeid = H5T_STD_I64LE;
355 else if (endianness == NC_ENDIAN_BIG)
356 typeid = H5T_STD_I64BE;
357 else
358 typeid = H5T_NATIVE_LLONG;
359 break;
360
361 case NC_UINT64:
362 if (endianness == NC_ENDIAN_LITTLE)
363 typeid = H5T_STD_U64LE;
364 else if (endianness == NC_ENDIAN_BIG)
365 typeid = H5T_STD_U64BE;
366 else
367 typeid = H5T_NATIVE_ULLONG;
368 break;
369
370 case NC_FLOAT:
371 if (endianness == NC_ENDIAN_LITTLE)
372 typeid = H5T_IEEE_F32LE;
373 else if (endianness == NC_ENDIAN_BIG)
374 typeid = H5T_IEEE_F32BE;
375 else
376 typeid = H5T_NATIVE_FLOAT;
377 break;
378
379 case NC_DOUBLE:
380 if (endianness == NC_ENDIAN_LITTLE)
381 typeid = H5T_IEEE_F64LE;
382 else if (endianness == NC_ENDIAN_BIG)
383 typeid = H5T_IEEE_F64BE;
384 else
385 typeid = H5T_NATIVE_DOUBLE;
386 break;
387
388 default:
389 /* Maybe this is a user defined type? */
390 if (nc4_find_type(h5, xtype, &type))
391 return NC_EBADTYPE;
392 if (!type)
393 return NC_EBADTYPE;
394 typeid = ((NC_HDF5_TYPE_INFO_T *)type->format_type_info)->hdf_typeid;
395 break;
396 }
397 assert(typeid);
398
399 /* Copy the HDF5 datatype, so the function operates uniformly */
400 if ((*hdf_typeid = H5Tcopy(typeid)) < 0)
401 return NC_EHDFERR;
402 typeid = 0;
403 }
404 assert(*hdf_typeid != -1);
405
406exit:
407 if (typeid > 0 && H5Tclose(typeid) < 0)
408 BAIL2(NC_EHDFERR);
409 return retval;
410}
411
426static int
427put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att)
428{
429 NC_HDF5_GRP_INFO_T *hdf5_grp;
430 hid_t datasetid = 0, locid;
431 hid_t attid = 0, spaceid = 0, file_typeid = 0;
432 hid_t existing_att_typeid = 0, existing_attid = 0, existing_spaceid = 0;
433 hsize_t dims[1]; /* netcdf attributes always 1-D. */
434 htri_t attr_exists;
435 void *data;
436 int phoney_data = 99;
437 int retval = NC_NOERR;
438
439 assert(att->hdr.name && grp && grp->format_grp_info);
440 LOG((3, "%s: varid %d att->hdr.id %d att->hdr.name %s att->nc_typeid %d "
441 "att->len %d", __func__, varid, att->hdr.id, att->hdr.name,
442 att->nc_typeid, att->len));
443
444 /* Get HDF5-specific group info. */
445 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
446
447 /* If the file is read-only, return an error. */
448 if (grp->nc4_info->no_write)
449 BAIL(NC_EPERM);
450
451 /* Get the hid to attach the attribute to, or read it from. */
452 if (varid == NC_GLOBAL)
453 locid = hdf5_grp->hdf_grpid;
454 else
455 {
456 if ((retval = nc4_open_var_grp2(grp, varid, &datasetid)))
457 BAIL(retval);
458 locid = datasetid;
459 }
460
461 /* Get the length ready, and find the HDF type we'll be
462 * writing. */
463 dims[0] = att->len;
464 if ((retval = nc4_get_hdf_typeid(grp->nc4_info, att->nc_typeid,
465 &file_typeid, 0)))
466 BAIL(retval);
467
468 /* Even if the length is zero, HDF5 won't let me write with a
469 * NULL pointer. So if the length of the att is zero, point to
470 * some phoney data (which won't be written anyway.)*/
471 if (!dims[0])
472 data = &phoney_data;
473#ifdef SEPDATA
474 else if (att->vldata)
475 data = att->vldata;
476 else if (att->stdata)
477 data = att->stdata;
478#endif
479 else
480 data = att->data;
481
482 /* NC_CHAR types require some extra work. The space ID is set to
483 * scalar, and the type is told how long the string is. If it's
484 * really zero length, set the size to 1. (The fact that it's
485 * really zero will be marked by the NULL dataspace, but HDF5
486 * doesn't allow me to set the size of the type to zero.)*/
487 if (att->nc_typeid == NC_CHAR)
488 {
489 size_t string_size = dims[0];
490 if (!string_size)
491 {
492 string_size = 1;
493 if ((spaceid = H5Screate(H5S_NULL)) < 0)
494 BAIL(NC_EATTMETA);
495 }
496 else
497 {
498 if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
499 BAIL(NC_EATTMETA);
500 }
501 if (H5Tset_size(file_typeid, string_size) < 0)
502 BAIL(NC_EATTMETA);
503 if (H5Tset_strpad(file_typeid, H5T_STR_NULLTERM) < 0)
504 BAIL(NC_EATTMETA);
505 }
506 else
507 {
508 if (!att->len)
509 {
510 if ((spaceid = H5Screate(H5S_NULL)) < 0)
511 BAIL(NC_EATTMETA);
512 }
513 else
514 {
515 if ((spaceid = H5Screate_simple(1, dims, NULL)) < 0)
516 BAIL(NC_EATTMETA);
517 }
518 }
519
520 /* Does the att exists already? */
521 if ((attr_exists = H5Aexists(locid, att->hdr.name)) < 0)
522 BAIL(NC_EHDFERR);
523 if (attr_exists)
524 {
525 hssize_t npoints;
526
527 /* Open the attribute. */
528 if ((existing_attid = H5Aopen(locid, att->hdr.name, H5P_DEFAULT)) < 0)
529 BAIL(NC_EATTMETA);
530
531 /* Find the type of the existing attribute. */
532 if ((existing_att_typeid = H5Aget_type(existing_attid)) < 0)
533 BAIL(NC_EATTMETA);
534
535 /* How big is the attribute? */
536 if ((existing_spaceid = H5Aget_space(existing_attid)) < 0)
537 BAIL(NC_EATTMETA);
538 if ((npoints = H5Sget_simple_extent_npoints(existing_spaceid)) < 0)
539 BAIL(NC_EATTMETA);
540
541 /* For text attributes the size is specified in the datatype
542 and it is enough to compare types using H5Tequal(). */
543 if (!H5Tequal(file_typeid, existing_att_typeid) ||
544 (att->nc_typeid != NC_CHAR && npoints != att->len))
545 {
546 /* The attribute exists but we cannot re-use it. */
547
548 /* Delete the attribute. */
549 if (H5Adelete(locid, att->hdr.name) < 0)
550 BAIL(NC_EHDFERR);
551
552 /* Re-create the attribute with the type and length
553 reflecting the new value (or values). */
554 if ((attid = H5Acreate1(locid, att->hdr.name, file_typeid, spaceid,
555 H5P_DEFAULT)) < 0)
556 BAIL(NC_EATTMETA);
557
558 /* Write the values, (even if length is zero). */
559 if (H5Awrite(attid, file_typeid, data) < 0)
560 BAIL(NC_EATTMETA);
561 }
562 else
563 {
564 /* The attribute exists and we can re-use it. */
565
566 /* Write the values, re-using the existing attribute. */
567 if (H5Awrite(existing_attid, file_typeid, data) < 0)
568 BAIL(NC_EATTMETA);
569 }
570 }
571 else
572 {
573 /* The attribute does not exist yet. */
574
575 /* Create the attribute. */
576 if ((attid = H5Acreate1(locid, att->hdr.name, file_typeid, spaceid,
577 H5P_DEFAULT)) < 0)
578 BAIL(NC_EATTMETA);
579
580 /* Write the values, (even if length is zero). */
581 if (H5Awrite(attid, file_typeid, data) < 0)
582 BAIL(NC_EATTMETA);
583 }
584
585exit:
586 if (file_typeid && H5Tclose(file_typeid))
587 BAIL2(NC_EHDFERR);
588 if (attid > 0 && H5Aclose(attid) < 0)
589 BAIL2(NC_EHDFERR);
590 if (existing_att_typeid && H5Tclose(existing_att_typeid))
591 BAIL2(NC_EHDFERR);
592 if (existing_attid > 0 && H5Aclose(existing_attid) < 0)
593 BAIL2(NC_EHDFERR);
594 if (spaceid > 0 && H5Sclose(spaceid) < 0)
595 BAIL2(NC_EHDFERR);
596 if (existing_spaceid > 0 && H5Sclose(existing_spaceid) < 0)
597 BAIL2(NC_EHDFERR);
598 return retval;
599}
600
612static int
613write_attlist(NCindex *attlist, int varid, NC_GRP_INFO_T *grp)
614{
615 NC_ATT_INFO_T *att;
616 int retval;
617 int i;
618
619 for(i = 0; i < ncindexsize(attlist); i++)
620 {
621 att = (NC_ATT_INFO_T *)ncindexith(attlist, i);
622 assert(att);
623 if (att->dirty)
624 {
625 LOG((4, "%s: writing att %s to varid %d", __func__, att->hdr.name, varid));
626 if ((retval = put_att_grpa(grp, varid, att)))
627 return retval;
628 att->dirty = NC_FALSE;
629 att->created = NC_TRUE;
630 }
631 }
632 return NC_NOERR;
633}
634
648static int
649write_coord_dimids(NC_VAR_INFO_T *var)
650{
651 NC_HDF5_VAR_INFO_T *hdf5_var;
652 hsize_t coords_len[1];
653 hid_t c_spaceid = -1, c_attid = -1;
654 int retval = NC_NOERR;
655
656 assert(var && var->format_var_info);
657
658 /* Get HDF5-specific var info. */
659 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
660
661 /* Set up space for attribute. */
662 coords_len[0] = var->ndims;
663 if ((c_spaceid = H5Screate_simple(1, coords_len, coords_len)) < 0)
664 BAIL(NC_EHDFERR);
665
666 /* Create the attribute. */
667 if ((c_attid = H5Acreate1(hdf5_var->hdf_datasetid, COORDINATES,
668 H5T_NATIVE_INT, c_spaceid, H5P_DEFAULT)) < 0)
669 BAIL(NC_EHDFERR);
670
671 /* Write our attribute. */
672 if (H5Awrite(c_attid, H5T_NATIVE_INT, var->dimids) < 0)
673 BAIL(NC_EHDFERR);
674
675exit:
676 if (c_spaceid >= 0 && H5Sclose(c_spaceid) < 0)
677 BAIL2(NC_EHDFERR);
678 if (c_attid >= 0 && H5Aclose(c_attid) < 0)
679 BAIL2(NC_EHDFERR);
680 return retval;
681}
682
695static int
696write_quantize_att(NC_VAR_INFO_T *var)
697{
698 NC_HDF5_VAR_INFO_T *hdf5_var;
699 hsize_t len = 1;
700 hid_t c_spaceid = -1, c_attid = -1;
701 char att_name[NC_MAX_NAME + 1];
702 int retval = NC_NOERR;
703
704 assert(var && var->format_var_info);
705
706 /* Get HDF5-specific var info. */
707 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
708
709 /* Different quantize algorithms get different attribute names. */
710 switch (var->quantize_mode)
711 {
713 sprintf(att_name, "%s", NC_QUANTIZE_BITGROOM_ATT_NAME);
714 break;
716 sprintf(att_name, "%s", NC_QUANTIZE_GRANULARBR_ATT_NAME);
717 break;
719 sprintf(att_name, "%s", NC_QUANTIZE_BITROUND_ATT_NAME);
720 break;
721 default:
722 return NC_EINVAL;
723 }
724
725 /* Set up space for attribute. */
726 if ((c_spaceid = H5Screate_simple(1, &len, &len)) < 0)
727 BAIL(NC_EHDFERR);
728
729 /* Create the attribute. */
730 if ((c_attid = H5Acreate1(hdf5_var->hdf_datasetid, att_name,
731 H5T_NATIVE_INT, c_spaceid, H5P_DEFAULT)) < 0)
732 BAIL(NC_EHDFERR);
733
734 /* Write our attribute. */
735 if (H5Awrite(c_attid, H5T_NATIVE_INT, &var->nsd) < 0)
736 BAIL(NC_EHDFERR);
737
738exit:
739 if (c_spaceid >= 0 && H5Sclose(c_spaceid) < 0)
740 BAIL2(NC_EHDFERR);
741 if (c_attid >= 0 && H5Aclose(c_attid) < 0)
742 BAIL2(NC_EHDFERR);
743 return retval;
744}
745
756static int
757write_netcdf4_dimid(hid_t datasetid, int dimid)
758{
759 hid_t dimid_spaceid = -1, dimid_attid = -1;
760 htri_t attr_exists;
761 int retval = NC_NOERR;
762
763 /* Create the space. */
764 if ((dimid_spaceid = H5Screate(H5S_SCALAR)) < 0)
765 BAIL(NC_EHDFERR);
766
767 /* Does the attribute already exist? If so, don't try to create it. */
768 if ((attr_exists = H5Aexists(datasetid, NC_DIMID_ATT_NAME)) < 0)
769 BAIL(NC_EHDFERR);
770 if (attr_exists)
771 dimid_attid = H5Aopen_by_name(datasetid, ".", NC_DIMID_ATT_NAME,
772 H5P_DEFAULT, H5P_DEFAULT);
773 else
774 /* Create the attribute if needed. */
775 dimid_attid = H5Acreate1(datasetid, NC_DIMID_ATT_NAME,
776 H5T_NATIVE_INT, dimid_spaceid, H5P_DEFAULT);
777 if (dimid_attid < 0)
778 BAIL(NC_EHDFERR);
779
780
781 /* Write it. */
782 LOG((4, "%s: writing secret dimid %d", __func__, dimid));
783 if (H5Awrite(dimid_attid, H5T_NATIVE_INT, &dimid) < 0)
784 BAIL(NC_EHDFERR);
785
786exit:
787 /* Close stuff*/
788 if (dimid_spaceid >= 0 && H5Sclose(dimid_spaceid) < 0)
789 BAIL2(NC_EHDFERR);
790 if (dimid_attid >= 0 && H5Aclose(dimid_attid) < 0)
791 BAIL2(NC_EHDFERR);
792
793 return retval;
794}
795
810static int
811var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid)
812{
813 NC_HDF5_GRP_INFO_T *hdf5_grp;
814 NC_HDF5_VAR_INFO_T *hdf5_var;
815 hid_t plistid = 0, access_plistid = 0, typeid = 0, spaceid = 0;
816 hsize_t chunksize[H5S_MAX_RANK], dimsize[H5S_MAX_RANK], maxdimsize[H5S_MAX_RANK];
817 int d;
818 void *fillp = NULL;
819 NC_DIM_INFO_T *dim = NULL;
820 char *name_to_use;
821 int retval;
822 unsigned int* params = NULL;
823
824 assert(grp && grp->format_grp_info && var && var->format_var_info);
825
826 LOG((3, "%s:: name %s", __func__, var->hdr.name));
827
828 /* Get HDF5-specific group and var info. */
829 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
830 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
831
832 /* Scalar or not, we need a creation property list. */
833 if ((plistid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
834 BAIL(NC_EHDFERR);
835 if ((access_plistid = H5Pcreate(H5P_DATASET_ACCESS)) < 0)
836 BAIL(NC_EHDFERR);
837
838 /* Turn off object tracking times in HDF5. */
839 if (H5Pset_obj_track_times(plistid, 0) < 0)
840 BAIL(NC_EHDFERR);
841
842 /* Find the HDF5 type of the dataset. */
843 if ((retval = nc4_get_hdf_typeid(grp->nc4_info, var->type_info->hdr.id, &typeid,
844 var->type_info->endianness)))
845 BAIL(retval);
846
847 /* Figure out what fill value to set, if any. */
848 if (var->no_fill)
849 {
850 /* Required to truly turn HDF5 fill values off */
851 if (H5Pset_fill_time(plistid, H5D_FILL_TIME_NEVER) < 0)
852 BAIL(NC_EHDFERR);
853 }
854 else
855 {
856 if ((retval = nc4_get_fill_value(grp->nc4_info, var, &fillp)))
857 BAIL(retval);
858
859 /* If there is a fill value, set it. */
860 if (fillp)
861 {
862 if (var->type_info->nc_type_class == NC_STRING)
863 {
864 if (H5Pset_fill_value(plistid, typeid, fillp) < 0)
865 BAIL(NC_EHDFERR);
866 }
867 else
868 {
869 /* The fill value set in HDF5 must always be presented as
870 * a native type, even if the endianness for this dataset
871 * is non-native. HDF5 will translate the fill value to
872 * the target endiannesss. */
873 hid_t fill_typeid = 0;
874
875 if ((retval = nc4_get_hdf_typeid(grp->nc4_info, var->type_info->hdr.id, &fill_typeid,
877 BAIL(retval);
878 if (H5Pset_fill_value(plistid, fill_typeid, fillp) < 0)
879 {
880 if (H5Tclose(fill_typeid) < 0)
881 BAIL(NC_EHDFERR);
882 BAIL(NC_EHDFERR);
883 }
884 if (H5Tclose(fill_typeid) < 0)
885 BAIL(NC_EHDFERR);
886 }
887 }
888 }
889
890 /* If the user wants to compress the data, using either zlib
891 * (a.k.a deflate) or szip, or another filter, set that up now.
892 * Szip and zip can be turned on
893 * either directly with nc_def_var_szip/deflate(), or using
894 * nc_def_var_filter(). If the user
895 * has specified a filter, it will be applied here. */
896 if(var->filters != NULL) {
897 int j;
898 NClist* filters = (NClist*)var->filters;
899 for(j=0;j<nclistlength(filters);j++) {
900 struct NC_HDF5_Filter* fi = (struct NC_HDF5_Filter*)nclistget(filters,j);
901 if(fi->filterid == H5Z_FILTER_FLETCHER32) {
902 if(H5Pset_fletcher32(plistid) < 0)
903 BAIL(NC_EHDFERR);
904 } else if(fi->filterid == H5Z_FILTER_SHUFFLE) {
905 if(H5Pset_shuffle(plistid) < 0)
906 BAIL(NC_EHDFERR);
907 } else if(fi->filterid == H5Z_FILTER_DEFLATE) {/* Handle zip case here */
908 unsigned level;
909 if(fi->nparams != 1)
910 BAIL(NC_EFILTER);
911 level = (int)fi->params[0];
912 if(H5Pset_deflate(plistid, level) < 0)
913 BAIL(NC_EFILTER);
914 } else if(fi->filterid == H5Z_FILTER_SZIP) {/* Handle szip case here */
915 int options_mask;
916 int bits_per_pixel;
917 if(fi->nparams != 2)
918 BAIL(NC_EFILTER);
919 options_mask = (int)fi->params[0];
920 bits_per_pixel = (int)fi->params[1];
921 if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0)
922 BAIL(NC_EFILTER);
923 } else {
924 herr_t code = H5Pset_filter(plistid, fi->filterid,
925#if 1
926 H5Z_FLAG_MANDATORY,
927#else
928 H5Z_FLAG_OPTIONAL,
929#endif
930 fi->nparams, fi->params);
931 if(code < 0)
932 BAIL(NC_EFILTER);
933 }
934 }
935 }
936
937 /* If ndims non-zero, get info for all dimensions. We look up the
938 dimids and get the len of each dimension. We need this to create
939 the space for the dataset. In netCDF a dimension length of zero
940 means an unlimited dimension. */
941 if (var->ndims)
942 {
943 int unlimdim = 0;
944
945 /* Check to see if any unlimited dimensions are used in this var. */
946 for (d = 0; d < var->ndims; d++) {
947 dim = var->dim[d];
948 assert(dim && dim->hdr.id == var->dimids[d]);
949 if (dim->unlimited)
950 unlimdim++;
951 }
952
953 /* If there are no unlimited dims, and no filters, and the user
954 * has not specified chunksizes, use contiguous variable for
955 * better performance. */
956 if (nclistlength((NClist*)var->filters) == 0 &&
957 (var->chunksizes == NULL || !var->chunksizes[0]) && !unlimdim)
958 var->storage = NC_CONTIGUOUS;
959
960 /* Gather current & maximum dimension sizes, along with chunk
961 * sizes. */
962 for (d = 0; d < var->ndims; d++)
963 {
964 dim = var->dim[d];
965 assert(dim && dim->hdr.id == var->dimids[d]);
966 dimsize[d] = dim->unlimited ? NC_HDF5_UNLIMITED_DIMSIZE : dim->len;
967 maxdimsize[d] = dim->unlimited ? H5S_UNLIMITED : (hsize_t)dim->len;
968 if (var->storage == NC_CHUNKED)
969 {
970 if (var->chunksizes[d])
971 chunksize[d] = var->chunksizes[d];
972 else
973 {
974 size_t type_size;
975 if (var->type_info->nc_type_class == NC_STRING)
976 type_size = sizeof(char *);
977 else
978 type_size = var->type_info->size;
979
980 /* Unlimited dim always gets chunksize of 1. */
981 if (dim->unlimited)
982 chunksize[d] = 1;
983 else
984 chunksize[d] = pow((double)DEFAULT_CHUNK_SIZE/type_size,
985 1/(double)(var->ndims - unlimdim));
986
987 /* If the chunksize is greater than the dim
988 * length, make it the dim length. */
989 if (!dim->unlimited && chunksize[d] > dim->len)
990 chunksize[d] = dim->len;
991
992 /* Remember the computed chunksize */
993 var->chunksizes[d] = chunksize[d];
994 }
995 }
996 }
997
998 /* Create the dataspace. */
999 if ((spaceid = H5Screate_simple(var->ndims, dimsize, maxdimsize)) < 0)
1000 BAIL(NC_EHDFERR);
1001 }
1002 else
1003 {
1004 if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
1005 BAIL(NC_EHDFERR);
1006 }
1007
1008 /* Set the var storage to contiguous, compact, or chunked. Don't
1009 * try to set chunking for scalar vars, they will default to
1010 * contiguous if not set to compact. */
1011 if (var->storage == NC_CONTIGUOUS)
1012 {
1013 if (H5Pset_layout(plistid, H5D_CONTIGUOUS) < 0)
1014 BAIL(NC_EHDFERR);
1015 }
1016 else if (var->storage == NC_COMPACT)
1017 {
1018 if (H5Pset_layout(plistid, H5D_COMPACT) < 0)
1019 BAIL(NC_EHDFERR);
1020 }
1021 else if (var->ndims)
1022 {
1023 if (H5Pset_chunk(plistid, var->ndims, chunksize) < 0)
1024 BAIL(NC_EHDFERR);
1025 }
1026
1027 /* Turn on creation order tracking. */
1028 if (!grp->nc4_info->no_attr_create_order) {
1029 if (H5Pset_attr_creation_order(plistid, H5P_CRT_ORDER_TRACKED|
1030 H5P_CRT_ORDER_INDEXED) < 0)
1031 BAIL(NC_EHDFERR);
1032 }
1033
1034 /* Set per-var chunk cache, for chunked datasets. */
1035 if (var->storage == NC_CHUNKED && var->chunkcache.size)
1036 if (H5Pset_chunk_cache(access_plistid, var->chunkcache.nelems,
1037 var->chunkcache.size, var->chunkcache.preemption) < 0)
1038 BAIL(NC_EHDFERR);
1039
1040 /* At long last, create the dataset. */
1041 name_to_use = var->alt_name ? var->alt_name : var->hdr.name;
1042 LOG((4, "%s: about to H5Dcreate2 dataset %s of type 0x%x", __func__,
1043 name_to_use, typeid));
1044 if ((hdf5_var->hdf_datasetid = H5Dcreate2(hdf5_grp->hdf_grpid, name_to_use, typeid,
1045 spaceid, H5P_DEFAULT, plistid, access_plistid)) < 0)
1046 BAIL(NC_EHDFERR);
1047 var->created = NC_TRUE;
1048 var->is_new_var = NC_FALSE;
1049
1050 /* Always write the hidden coordinates attribute, which lists the
1051 * dimids of this var. When present, this speeds opens. When not
1052 * present, dimscale matching is used. */
1053 if (var->ndims)
1054 if ((retval = write_coord_dimids(var)))
1055 BAIL(retval);
1056
1057 /* If this is a dimscale, mark it as such in the HDF5 file. Also
1058 * find the dimension info and store the dataset id of the dimscale
1059 * dataset. */
1060 if (hdf5_var->dimscale)
1061 {
1062 if (H5DSset_scale(hdf5_var->hdf_datasetid, var->hdr.name) < 0)
1063 BAIL(NC_EHDFERR);
1064
1065 /* If this is a multidimensional coordinate variable, write a
1066 * coordinates attribute. */
1067 /* if (var->ndims > 1) */
1068 /* if ((retval = write_coord_dimids(var))) */
1069 /* BAIL(retval); */
1070
1071 /* If desired, write the netCDF dimid. */
1072 if (write_dimid)
1073 if ((retval = write_netcdf4_dimid(hdf5_var->hdf_datasetid, var->dimids[0])))
1074 BAIL(retval);
1075 }
1076
1077 /* If quantization is in use, write an attribute indicating it, a
1078 * single integer which is the number of significant digits
1079 * (NSD, for BitGroom and Granular BitRound) or number of significant bits
1080 * (NSB, for BitRound). */
1081 if (var->quantize_mode)
1082 if ((retval = write_quantize_att(var)))
1083 BAIL(retval);
1084
1085 /* Write attributes for this var. */
1086 if ((retval = write_attlist(var->att, var->hdr.id, grp)))
1087 BAIL(retval);
1088
1089 /* The file is now up-to-date with all settings for this var. */
1090 var->attr_dirty = NC_FALSE;
1091
1092exit:
1093 nullfree(params);
1094 if (typeid > 0 && H5Tclose(typeid) < 0)
1095 BAIL2(NC_EHDFERR);
1096 if (plistid > 0 && H5Pclose(plistid) < 0)
1097 BAIL2(NC_EHDFERR);
1098 if (access_plistid > 0 && H5Pclose(access_plistid) < 0)
1099 BAIL2(NC_EHDFERR);
1100 if (spaceid > 0 && H5Sclose(spaceid) < 0)
1101 BAIL2(NC_EHDFERR);
1102 if (fillp)
1103 {
1104 if (var->type_info->nc_type_class == NC_VLEN)
1105 nc_free_vlen((nc_vlen_t *)fillp);
1106 else if (var->type_info->nc_type_class == NC_STRING && *(char **)fillp)
1107 free(*(char **)fillp);
1108 free(fillp);
1109 }
1110
1111 return retval;
1112}
1113
1127int
1128nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
1129{
1130 size_t chunk_size_bytes = 1;
1131 int d;
1132 int retval;
1133
1134 /* Nothing to be done for contiguous or compact data. */
1135 if (var->storage != NC_CHUNKED)
1136 return NC_NOERR;
1137
1138#ifdef USE_PARALLEL4
1139 /* Don't set cache for files using parallel I/O. */
1140 if (grp->nc4_info->parallel)
1141 return NC_NOERR;
1142#endif
1143
1144 /* How many bytes in the chunk? */
1145 for (d = 0; d < var->ndims; d++)
1146 chunk_size_bytes *= var->chunksizes[d];
1147 if (var->type_info->size)
1148 chunk_size_bytes *= var->type_info->size;
1149 else
1150 chunk_size_bytes *= sizeof(char *);
1151
1152 /* If the chunk cache is too small, and the user has not changed
1153 * the default value of the chunk cache size, then increase the
1154 * size of the cache. */
1155 if (var->chunkcache.size == CHUNK_CACHE_SIZE)
1156 if (chunk_size_bytes > var->chunkcache.size)
1157 {
1158 var->chunkcache.size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE;
1159 if (var->chunkcache.size > MAX_DEFAULT_CACHE_SIZE)
1160 var->chunkcache.size = MAX_DEFAULT_CACHE_SIZE;
1161 if ((retval = nc4_reopen_dataset(grp, var)))
1162 return retval;
1163 }
1164
1165 return NC_NOERR;
1166}
1167
1183static int
1184commit_type(NC_GRP_INFO_T *grp, NC_TYPE_INFO_T *type)
1185{
1186 NC_HDF5_GRP_INFO_T *hdf5_grp;
1187 NC_HDF5_TYPE_INFO_T *hdf5_type;
1188 hid_t base_hdf_typeid;
1189 int retval;
1190
1191 assert(grp && grp->format_grp_info && type && type->format_type_info);
1192
1193 /* Get HDF5-specific group and type info. */
1194 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1195 hdf5_type = (NC_HDF5_TYPE_INFO_T *)type->format_type_info;
1196
1197 /* Did we already record this type? */
1198 if (type->committed)
1199 return NC_NOERR;
1200
1201 /* Is this a compound type? */
1202 if (type->nc_type_class == NC_COMPOUND)
1203 {
1204 NC_FIELD_INFO_T *field;
1205 hid_t hdf_base_typeid, hdf_typeid;
1206 int i;
1207
1208 if ((hdf5_type->hdf_typeid = H5Tcreate(H5T_COMPOUND, type->size)) < 0)
1209 return NC_EHDFERR;
1210 LOG((4, "creating compound type %s hdf_typeid 0x%x", type->hdr.name,
1211 hdf5_type->hdf_typeid));
1212
1213 for(i=0;i<nclistlength(type->u.c.field);i++)
1214 {
1215 field = (NC_FIELD_INFO_T *)nclistget(type->u.c.field, i);
1216 assert(field);
1217 if ((retval = nc4_get_hdf_typeid(grp->nc4_info, field->nc_typeid,
1218 &hdf_base_typeid, type->endianness)))
1219 return retval;
1220
1221 /* If this is an array, create a special array type. */
1222 if (field->ndims)
1223 {
1224 int d;
1225 hsize_t dims[NC_MAX_VAR_DIMS];
1226
1227 for (d = 0; d < field->ndims; d++)
1228 dims[d] = field->dim_size[d];
1229 if ((hdf_typeid = H5Tarray_create1(hdf_base_typeid, field->ndims,
1230 dims, NULL)) < 0)
1231 {
1232 if (H5Tclose(hdf_base_typeid) < 0)
1233 return NC_EHDFERR;
1234 return NC_EHDFERR;
1235 }
1236 if (H5Tclose(hdf_base_typeid) < 0)
1237 return NC_EHDFERR;
1238 }
1239 else
1240 hdf_typeid = hdf_base_typeid;
1241 LOG((4, "inserting field %s offset %d hdf_typeid 0x%x", field->hdr.name,
1242 field->offset, hdf_typeid));
1243 if (H5Tinsert(hdf5_type->hdf_typeid, field->hdr.name, field->offset,
1244 hdf_typeid) < 0)
1245 return NC_EHDFERR;
1246 if (H5Tclose(hdf_typeid) < 0)
1247 return NC_EHDFERR;
1248 }
1249 }
1250 else if (type->nc_type_class == NC_VLEN)
1251 {
1252 /* Find the HDF typeid of the base type of this vlen. */
1253 if ((retval = nc4_get_hdf_typeid(grp->nc4_info, type->u.v.base_nc_typeid,
1254 &base_hdf_typeid, type->endianness)))
1255 return retval;
1256
1257 /* Create a vlen type. */
1258 if ((hdf5_type->hdf_typeid = H5Tvlen_create(base_hdf_typeid)) < 0)
1259 return NC_EHDFERR;
1260 }
1261 else if (type->nc_type_class == NC_OPAQUE)
1262 {
1263 /* Create the opaque type. */
1264 if ((hdf5_type->hdf_typeid = H5Tcreate(H5T_OPAQUE, type->size)) < 0)
1265 return NC_EHDFERR;
1266 }
1267 else if (type->nc_type_class == NC_ENUM)
1268 {
1269 NC_ENUM_MEMBER_INFO_T *enum_m;
1270 int i;
1271
1272 if (nclistlength(type->u.e.enum_member) == 0)
1273 return NC_EINVAL;
1274
1275 /* Find the HDF typeid of the base type of this enum. */
1276 if ((retval = nc4_get_hdf_typeid(grp->nc4_info, type->u.e.base_nc_typeid,
1277 &base_hdf_typeid, type->endianness)))
1278 return retval;
1279
1280 /* Create an enum type. */
1281 if ((hdf5_type->hdf_typeid = H5Tenum_create(base_hdf_typeid)) < 0)
1282 return NC_EHDFERR;
1283
1284 /* Add all the members to the HDF5 type. */
1285 for(i=0;i<nclistlength(type->u.e.enum_member);i++) {
1286 enum_m = (NC_ENUM_MEMBER_INFO_T*)nclistget(type->u.e.enum_member,i);
1287 if (H5Tenum_insert(hdf5_type->hdf_typeid, enum_m->name, enum_m->value) < 0)
1288 return NC_EHDFERR;
1289 }
1290 }
1291 else
1292 {
1293 LOG((0, "Unknown class: %d", type->nc_type_class));
1294 return NC_EBADTYPE;
1295 }
1296
1297 /* Commit the type. */
1298 if (H5Tcommit1(hdf5_grp->hdf_grpid, type->hdr.name, hdf5_type->hdf_typeid) < 0)
1299 return NC_EHDFERR;
1300 type->committed = NC_TRUE;
1301 LOG((4, "just committed type %s, HDF typeid: 0x%x", type->hdr.name,
1302 hdf5_type->hdf_typeid));
1303
1304 /* Later we will always use the native typeid. In this case, it is
1305 * a copy of the same type pointed to by hdf_typeid, but it's
1306 * easier to maintain a copy. */
1307 if ((hdf5_type->native_hdf_typeid = H5Tget_native_type(hdf5_type->hdf_typeid,
1308 H5T_DIR_DEFAULT)) < 0)
1309 return NC_EHDFERR;
1310
1311 return NC_NOERR;
1312}
1313
1324static int
1325write_nc3_strict_att(hid_t hdf_grpid)
1326{
1327 hid_t attid = 0, spaceid = 0;
1328 int one = 1;
1329 int retval = NC_NOERR;
1330 htri_t attr_exists;
1331
1332 /* If the attribute already exists, call that a success and return
1333 * NC_NOERR. */
1334 if ((attr_exists = H5Aexists(hdf_grpid, NC3_STRICT_ATT_NAME)) < 0)
1335 return NC_EHDFERR;
1336 if (attr_exists)
1337 return NC_NOERR;
1338
1339 /* Create the attribute to mark this as a file that needs to obey
1340 * strict netcdf-3 rules. */
1341 if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
1342 BAIL(NC_EFILEMETA);
1343 if ((attid = H5Acreate1(hdf_grpid, NC3_STRICT_ATT_NAME,
1344 H5T_NATIVE_INT, spaceid, H5P_DEFAULT)) < 0)
1345 BAIL(NC_EFILEMETA);
1346 if (H5Awrite(attid, H5T_NATIVE_INT, &one) < 0)
1347 BAIL(NC_EFILEMETA);
1348
1349exit:
1350 if (spaceid > 0 && (H5Sclose(spaceid) < 0))
1351 BAIL2(NC_EFILEMETA);
1352 if (attid > 0 && (H5Aclose(attid) < 0))
1353 BAIL2(NC_EFILEMETA);
1354 return retval;
1355}
1356
1369static int
1370create_group(NC_GRP_INFO_T *grp)
1371{
1372 NC_HDF5_GRP_INFO_T *hdf5_grp, *parent_hdf5_grp;
1373 hid_t gcpl_id = -1;
1374 int retval = NC_NOERR;;
1375
1376 assert(grp && grp->format_grp_info && grp->parent &&
1377 grp->parent->format_grp_info);
1378
1379 /* Get HDF5 specific group info for group and parent. */
1380 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1381 parent_hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->parent->format_grp_info;
1382 assert(parent_hdf5_grp->hdf_grpid);
1383
1384 /* Create group, with link_creation_order set in the group
1385 * creation property list. */
1386 if ((gcpl_id = H5Pcreate(H5P_GROUP_CREATE)) < 0)
1387 BAIL(NC_EHDFERR);
1388
1389 /* Set track_times to be FALSE. */
1390 if (H5Pset_obj_track_times(gcpl_id, 0) < 0)
1391 BAIL(NC_EHDFERR);
1392
1393 /* Tell HDF5 to keep track of objects in creation order. */
1394 if (H5Pset_link_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED|H5P_CRT_ORDER_INDEXED) < 0)
1395 BAIL(NC_EHDFERR);
1396
1397 /* Tell HDF5 to keep track of attributes in creation order. */
1398 if (!grp->nc4_info->no_attr_create_order) {
1399 if (H5Pset_attr_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED|H5P_CRT_ORDER_INDEXED) < 0)
1400 BAIL(NC_EHDFERR);
1401 }
1402
1403 /* Create the group. */
1404 if ((hdf5_grp->hdf_grpid = H5Gcreate2(parent_hdf5_grp->hdf_grpid, grp->hdr.name,
1405 H5P_DEFAULT, gcpl_id, H5P_DEFAULT)) < 0)
1406 BAIL(NC_EHDFERR);
1407
1408exit:
1409 if (gcpl_id > -1 && H5Pclose(gcpl_id) < 0)
1410 BAIL2(NC_EHDFERR);
1411 if (retval)
1412 if (hdf5_grp->hdf_grpid > 0 && H5Gclose(hdf5_grp->hdf_grpid) < 0)
1413 BAIL2(NC_EHDFERR);
1414 return retval;
1415}
1416
1429static int
1430attach_dimscales(NC_GRP_INFO_T *grp)
1431{
1432 NC_VAR_INFO_T *var;
1433 NC_HDF5_VAR_INFO_T *hdf5_var;
1434 int d, v;
1435
1436 /* Attach dimension scales. */
1437 for (v = 0; v < ncindexsize(grp->vars); v++)
1438 {
1439 /* Get pointer to var and HDF5-specific var info. */
1440 var = (NC_VAR_INFO_T *)ncindexith(grp->vars, v);
1441 assert(var && var->format_var_info);
1442 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
1443
1444 /* Scales themselves do not attach. But I really wish they
1445 * would. */
1446 if (hdf5_var->dimscale)
1447 continue;
1448
1449 /* Find the scale for each dimension, if any, and attach it. */
1450 for (d = 0; d < var->ndims; d++)
1451 {
1452 /* Is there a dimscale for this dimension? */
1453 if (hdf5_var->dimscale_attached)
1454 {
1455 if (!hdf5_var->dimscale_attached[d])
1456 {
1457 hid_t dsid; /* Dataset ID for dimension */
1458 assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d] &&
1459 var->dim[d]->format_dim_info);
1460
1461 LOG((2, "%s: attaching scale for dimid %d to var %s",
1462 __func__, var->dimids[d], var->hdr.name));
1463
1464 /* Find dataset ID for dimension */
1465 if (var->dim[d]->coord_var)
1466 dsid = ((NC_HDF5_VAR_INFO_T *)(var->dim[d]->coord_var->format_var_info))->hdf_datasetid;
1467 else
1468 dsid = ((NC_HDF5_DIM_INFO_T *)var->dim[d]->format_dim_info)->hdf_dimscaleid;
1469 assert(dsid > 0);
1470
1471 /* Attach the scale. */
1472 if (H5DSattach_scale(hdf5_var->hdf_datasetid, dsid, d) < 0)
1473 return NC_EDIMSCALE;
1474 hdf5_var->dimscale_attached[d] = NC_TRUE;
1475 }
1476 }
1477 }
1478 }
1479
1480 return NC_NOERR;
1481}
1482
1493static int
1494var_exists(hid_t grpid, char *name, nc_bool_t *exists)
1495{
1496 htri_t link_exists;
1497
1498 /* Reset the boolean */
1499 *exists = NC_FALSE;
1500
1501 /* Check if the object name exists in the group */
1502 if ((link_exists = H5Lexists(grpid, name, H5P_DEFAULT)) < 0)
1503 return NC_EHDFERR;
1504 if (link_exists)
1505 {
1506 H5G_stat_t statbuf;
1507
1508 /* Get info about the object */
1509 if (H5Gget_objinfo(grpid, name, 1, &statbuf) < 0)
1510 return NC_EHDFERR;
1511
1512 if (H5G_DATASET == statbuf.type)
1513 *exists = NC_TRUE;
1514 }
1515
1516 return NC_NOERR;
1517}
1518
1534static int
1535remove_coord_atts(hid_t hdf_datasetid)
1536{
1537 htri_t attr_exists;
1538
1539 /* If the variable dataset has an optional NC_DIMID_ATT_NAME
1540 * attribute, delete it. */
1541 if ((attr_exists = H5Aexists(hdf_datasetid, NC_DIMID_ATT_NAME)) < 0)
1542 return NC_EHDFERR;
1543 if (attr_exists)
1544 {
1545 if (H5Adelete(hdf_datasetid, NC_DIMID_ATT_NAME) < 0)
1546 return NC_EHDFERR;
1547 }
1548
1549 /* Remove the dimension scale 'CLASS' & 'NAME' attributes. */
1550 if ((attr_exists = H5Aexists(hdf_datasetid,
1551 HDF5_DIMSCALE_CLASS_ATT_NAME)) < 0)
1552 return NC_EHDFERR;
1553 if (attr_exists)
1554 {
1555 if (H5Adelete(hdf_datasetid, HDF5_DIMSCALE_CLASS_ATT_NAME) < 0)
1556 return NC_EHDFERR;
1557 }
1558 if ((attr_exists = H5Aexists(hdf_datasetid,
1559 HDF5_DIMSCALE_NAME_ATT_NAME)) < 0)
1560 return NC_EHDFERR;
1561 if (attr_exists)
1562 {
1563 if (H5Adelete(hdf_datasetid, HDF5_DIMSCALE_NAME_ATT_NAME) < 0)
1564 return NC_EHDFERR;
1565 }
1566 return NC_NOERR;
1567}
1568
1583static int
1584write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid)
1585{
1586 NC_HDF5_GRP_INFO_T *hdf5_grp;
1587 NC_HDF5_VAR_INFO_T *hdf5_var;
1588 nc_bool_t replace_existing_var = NC_FALSE;
1589 int retval;
1590
1591 assert(var && var->format_var_info && grp && grp->format_grp_info);
1592
1593 LOG((4, "%s: writing var %s", __func__, var->hdr.name));
1594
1595 /* Get HDF5-specific group and var info. */
1596 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1597 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
1598
1599 /* If the variable has already been created & the fill value changed,
1600 * indicate that the existing variable should be replaced. */
1601 if (var->created && var->fill_val_changed)
1602 {
1603 replace_existing_var = NC_TRUE;
1604 var->fill_val_changed = NC_FALSE;
1605 /* If the variable is going to be replaced, we need to flag any
1606 other attributes associated with the variable as 'dirty', or
1607 else *only* the fill value attribute will be copied over and
1608 the rest will be lost. See
1609 https://github.com/Unidata/netcdf-c/issues/239 */
1610 flag_atts_dirty(var->att);
1611 }
1612
1613 /* Is this a coordinate var that has already been created in
1614 * the HDF5 file as a dimscale dataset? Check for dims with the
1615 * same name in this group. If there is one, check to see if
1616 * this object exists in the HDF group. */
1617 if (var->became_coord_var)
1618 {
1619 if ((NC_DIM_INFO_T *)ncindexlookup(grp->dim, var->hdr.name))
1620 {
1621 nc_bool_t exists;
1622
1623 if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists)))
1624 return retval;
1625 if (exists)
1626 {
1627 /* Indicate that the variable already exists, and should
1628 * be replaced. */
1629 replace_existing_var = NC_TRUE;
1630 flag_atts_dirty(var->att);
1631 }
1632 }
1633 }
1634
1635 /* Check dims if the variable will be replaced, so that the
1636 * dimensions will be de-attached and re-attached correctly. */
1637 if (replace_existing_var)
1638 {
1639 NC_DIM_INFO_T *d1;
1640
1641 /* Is there a dim with this var's name? */
1642 if ((d1 = (NC_DIM_INFO_T *)ncindexlookup(grp->dim, var->hdr.name)))
1643 {
1644 nc_bool_t exists;
1645 assert(d1->format_dim_info && d1->hdr.name);
1646
1647 if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists)))
1648 return retval;
1649 if (exists)
1650 {
1651 hid_t dsid;
1652
1653 /* Find dataset ID for dimension */
1654 if (d1->coord_var)
1655 dsid = ((NC_HDF5_VAR_INFO_T *)d1->coord_var->format_var_info)->hdf_datasetid;
1656 else
1657 dsid = ((NC_HDF5_DIM_INFO_T *)d1->format_dim_info)->hdf_dimscaleid;
1658 assert(dsid > 0);
1659
1660 /* If we're replacing an existing dimscale dataset, go to
1661 * every var in the file and detach this dimension scale,
1662 * because we have to delete it. */
1663 if ((retval = rec_detach_scales(grp->nc4_info->root_grp,
1664 var->dimids[0], dsid)))
1665 return retval;
1666 }
1667 }
1668 }
1669
1670 /* If this is not a dimension scale, remove any attached scales,
1671 * and delete dimscale attributes from the var. */
1672 if (var->was_coord_var && hdf5_var->dimscale_attached)
1673 {
1674 int d;
1675
1676 /* If the variable already exists in the file, Remove any dimension scale
1677 * attributes from it, if they exist. */
1678 if (var->created)
1679 if ((retval = remove_coord_atts(hdf5_var->hdf_datasetid)))
1680 return retval;
1681
1682 /* If this is a regular var, detach all its dim scales. */
1683 for (d = 0; d < var->ndims; d++)
1684 {
1685 if (hdf5_var->dimscale_attached[d])
1686 {
1687 hid_t dsid; /* Dataset ID for dimension */
1688 assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d] &&
1689 var->dim[d]->format_dim_info);
1690
1691 /* Find dataset ID for dimension */
1692 if (var->dim[d]->coord_var)
1693 dsid = ((NC_HDF5_VAR_INFO_T *)var->dim[d]->coord_var->format_var_info)->hdf_datasetid;
1694 else
1695 dsid = ((NC_HDF5_DIM_INFO_T *)var->dim[d]->format_dim_info)->hdf_dimscaleid;
1696 assert(dsid > 0);
1697
1698 /* Detach this dim scale. */
1699 if (H5DSdetach_scale(hdf5_var->hdf_datasetid, dsid, d) < 0)
1700 return NC_EHDFERR;
1701 hdf5_var->dimscale_attached[d] = NC_FALSE;
1702 }
1703 }
1704 }
1705
1706 /* Delete the HDF5 dataset that is to be replaced. */
1707 if (replace_existing_var)
1708 {
1709 /* Free the HDF5 dataset id. */
1710 if (hdf5_var->hdf_datasetid && H5Dclose(hdf5_var->hdf_datasetid) < 0)
1711 return NC_EHDFERR;
1712 hdf5_var->hdf_datasetid = 0;
1713
1714 /* Now delete the variable. */
1715 if (H5Gunlink(hdf5_grp->hdf_grpid, var->hdr.name) < 0)
1716 return NC_EDIMMETA;
1717 }
1718
1719 /* Create the dataset. */
1720 if (var->is_new_var || replace_existing_var)
1721 {
1722 if ((retval = var_create_dataset(grp, var, write_dimid)))
1723 return retval;
1724 }
1725 else
1726 {
1727 if (write_dimid && var->ndims)
1728 if ((retval = write_netcdf4_dimid(hdf5_var->hdf_datasetid,
1729 var->dimids[0])))
1730 return retval;
1731 }
1732
1733 if (replace_existing_var)
1734 {
1735 /* If this is a dimension scale, reattach the scale everywhere it
1736 * is used. (Recall that netCDF dimscales are always 1-D). */
1737 if(hdf5_var->dimscale)
1738 {
1739 if ((retval = rec_reattach_scales(grp->nc4_info->root_grp,
1740 var->dimids[0], hdf5_var->hdf_datasetid)))
1741 return retval;
1742 }
1743 /* If it's not a dimension scale, clear the dimscale attached flags,
1744 * so the dimensions are re-attached. */
1745 else
1746 {
1747 if (hdf5_var->dimscale_attached)
1748 memset(hdf5_var->dimscale_attached, 0, sizeof(nc_bool_t) * var->ndims);
1749 }
1750 }
1751
1752 /* Clear coord. var state transition flags */
1753 var->was_coord_var = NC_FALSE;
1754 var->became_coord_var = NC_FALSE;
1755
1756 /* Now check the attributes for this var. */
1757 if (var->attr_dirty)
1758 {
1759 /* Write attributes for this var. */
1760 if ((retval = write_attlist(var->att, var->hdr.id, grp)))
1761 return retval;
1762 var->attr_dirty = NC_FALSE;
1763 }
1764
1765 return NC_NOERR;
1766}
1767
1781int
1782nc4_create_dim_wo_var(NC_DIM_INFO_T *dim)
1783{
1784 NC_HDF5_DIM_INFO_T *hdf5_dim;
1785 NC_HDF5_GRP_INFO_T *hdf5_grp;
1786 hid_t spaceid = -1, create_propid = -1;
1787 hsize_t dims[1], max_dims[1], chunk_dims[1] = {1};
1788 char dimscale_wo_var[NC_MAX_NAME];
1789 int retval = NC_NOERR;
1790
1791 LOG((4, "%s: creating dim %s", __func__, dim->hdr.name));
1792
1793 /* Sanity check */
1794 assert(!dim->coord_var);
1795
1796 /* Get HDF5-specific dim and group info. */
1797 hdf5_grp = (NC_HDF5_GRP_INFO_T *)dim->container->format_grp_info;
1798 hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1799
1800 /* Create a property list. */
1801 if ((create_propid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
1802 BAIL(NC_EHDFERR);
1803
1804 /* Turn off recording of times associated with this object. */
1805 if (H5Pset_obj_track_times(create_propid, 0) < 0)
1806 BAIL(NC_EHDFERR);
1807
1808 /* Set size of dataset to size of dimension. */
1809 dims[0] = dim->len;
1810 max_dims[0] = dim->len;
1811
1812 /* If this dimension scale is unlimited (i.e. it's an unlimited
1813 * dimension), then set up chunking, with a chunksize of 1. */
1814 if (dim->unlimited)
1815 {
1816 max_dims[0] = H5S_UNLIMITED;
1817 if (H5Pset_chunk(create_propid, 1, chunk_dims) < 0)
1818 BAIL(NC_EHDFERR);
1819 }
1820
1821 /* Set up space. */
1822 if ((spaceid = H5Screate_simple(1, dims, max_dims)) < 0)
1823 BAIL(NC_EHDFERR);
1824
1825 /* Turn on creation-order tracking. */
1826 if (!dim->container->nc4_info->no_attr_create_order) {
1827 if (H5Pset_attr_creation_order(create_propid, H5P_CRT_ORDER_TRACKED|
1828 H5P_CRT_ORDER_INDEXED) < 0)
1829 BAIL(NC_EHDFERR);
1830 }
1831 /* Create the dataset that will be the dimension scale. */
1832 LOG((4, "%s: about to H5Dcreate1 a dimscale dataset %s", __func__,
1833 dim->hdr.name));
1834 if ((hdf5_dim->hdf_dimscaleid = H5Dcreate2(hdf5_grp->hdf_grpid, dim->hdr.name,
1835 H5T_IEEE_F32BE, spaceid,
1836 H5P_DEFAULT, create_propid,
1837 H5P_DEFAULT)) < 0)
1838 BAIL(NC_EHDFERR);
1839
1840 /* Indicate that this is a scale. Also indicate that not
1841 * be shown to the user as a variable. It is hidden. It is
1842 * a DIM WITHOUT A VARIABLE! */
1843 sprintf(dimscale_wo_var, "%s%10d", DIM_WITHOUT_VARIABLE, (int)dim->len);
1844 if (H5DSset_scale(hdf5_dim->hdf_dimscaleid, dimscale_wo_var) < 0)
1845 BAIL(NC_EHDFERR);
1846
1847 /* Since this dimension was created out of order, we cannot rely on
1848 * it getting the correct dimid on file open. We must assign it
1849 * explicitly. */
1850 if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id)))
1851 BAIL(retval);
1852
1853exit:
1854 if (spaceid > 0 && H5Sclose(spaceid) < 0)
1855 BAIL2(NC_EHDFERR);
1856 if (create_propid > 0 && H5Pclose(create_propid) < 0)
1857 BAIL2(NC_EHDFERR);
1858 return retval;
1859}
1860
1873static int
1874write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid)
1875{
1876 NC_HDF5_DIM_INFO_T *hdf5_dim;
1877 int retval = NC_NOERR;
1878
1879 assert(dim && dim->format_dim_info && grp && grp->format_grp_info);
1880
1881 /* Get HDF5-specific dim and group info. */
1882 hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1883
1884 /* If there's no dimscale dataset for this dim, create one,
1885 * and mark that it should be hidden from netCDF as a
1886 * variable. (That is, it should appear as a dimension
1887 * without an associated variable.) */
1888 if (!hdf5_dim->hdf_dimscaleid)
1889 if ((retval = nc4_create_dim_wo_var(dim)))
1890 BAIL(retval);
1891
1892 /* Did we extend an unlimited dimension? */
1893 if (dim->extended)
1894 {
1895 NC_VAR_INFO_T *v1 = NULL;
1896
1897 assert(dim->unlimited);
1898
1899 /* If this is a dimension with an associated coordinate var,
1900 * then update the length of that coord var. */
1901 v1 = dim->coord_var;
1902 if (v1)
1903 {
1904 NC_HDF5_VAR_INFO_T *hdf5_v1;
1905 hsize_t *new_size;
1906 int d1;
1907
1908 hdf5_v1 = (NC_HDF5_VAR_INFO_T *)v1->format_var_info;
1909
1910 /* Extend the dimension scale dataset to reflect the new
1911 * length of the dimension. */
1912 if (!(new_size = malloc(v1->ndims * sizeof(hsize_t))))
1913 BAIL(NC_ENOMEM);
1914 for (d1 = 0; d1 < v1->ndims; d1++)
1915 {
1916 assert(v1->dim[d1] && v1->dim[d1]->hdr.id == v1->dimids[d1]);
1917 new_size[d1] = v1->dim[d1]->len;
1918 }
1919 if (H5Dset_extent(hdf5_v1->hdf_datasetid, new_size) < 0)
1920 BAIL(NC_EHDFERR);
1921 free(new_size);
1922 }
1923 }
1924
1925 /* If desired, write the secret dimid. This will be used instead of
1926 * the dimid that the dimension would otherwise receive based on
1927 * creation order. This can be necessary when dims and their
1928 * coordinate variables were created in different order. */
1929 if (write_dimid && hdf5_dim->hdf_dimscaleid)
1930 if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id)))
1931 BAIL(retval);
1932
1933exit:
1934
1935 return retval;
1936}
1937
1950int
1951nc4_rec_write_metadata(NC_GRP_INFO_T *grp, nc_bool_t bad_coord_order)
1952{
1953 NC_DIM_INFO_T *dim = NULL;
1954 NC_VAR_INFO_T *var = NULL;
1955 NC_GRP_INFO_T *child_grp = NULL;
1956 int coord_varid = -1;
1957 int var_index = 0;
1958 int dim_index = 0;
1959 int retval;
1960 int i;
1961
1962 assert(grp && grp->hdr.name &&
1963 ((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid);
1964 LOG((3, "%s: grp->hdr.name %s, bad_coord_order %d", __func__, grp->hdr.name,
1965 bad_coord_order));
1966
1967 /* Write global attributes for this group. */
1968 if ((retval = write_attlist(grp->att, NC_GLOBAL, grp)))
1969 return retval;
1970
1971 /* Set the pointers to the beginning of the list of dims & vars in this
1972 * group. */
1973 dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, dim_index);
1974 var = (NC_VAR_INFO_T *)ncindexith(grp->vars, var_index);
1975
1976 /* Because of HDF5 ordering the dims and vars have to be stored in
1977 * this way to ensure that the dims and coordinate vars come out in
1978 * the correct order. */
1979 while (dim || var)
1980 {
1981 nc_bool_t found_coord, wrote_coord;
1982
1983 /* Write non-coord dims in order, stopping at the first one that
1984 * has an associated coord var. */
1985 for (found_coord = NC_FALSE; dim && !found_coord; )
1986 {
1987 if (!dim->coord_var)
1988 {
1989 if ((retval = write_dim(dim, grp, bad_coord_order)))
1990 return retval;
1991 }
1992 else
1993 {
1994 coord_varid = dim->coord_var->hdr.id;
1995 found_coord = NC_TRUE;
1996 }
1997 dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, ++dim_index);
1998 }
1999
2000 /* Write each var. When we get to the coord var we are waiting
2001 * for (if any), then we break after writing it. */
2002 for (wrote_coord = NC_FALSE; var && !wrote_coord; )
2003 {
2004 if ((retval = write_var(var, grp, bad_coord_order)))
2005 return retval;
2006 if (found_coord && var->hdr.id == coord_varid)
2007 wrote_coord = NC_TRUE;
2008 var = (NC_VAR_INFO_T *)ncindexith(grp->vars, ++var_index);
2009 }
2010 } /* end while */
2011
2012 /* Attach dimscales to vars in this group. Unless directed not to. */
2013 if (!grp->nc4_info->no_dimscale_attach) {
2014 if ((retval = attach_dimscales(grp)))
2015 return retval;
2016 }
2017
2018 /* If there are any child groups, write their metadata. */
2019 for (i = 0; i < ncindexsize(grp->children); i++)
2020 {
2021 child_grp = (NC_GRP_INFO_T *)ncindexith(grp->children, i);
2022 assert(child_grp);
2023 if ((retval = nc4_rec_write_metadata(child_grp, bad_coord_order)))
2024 return retval;
2025 }
2026 return NC_NOERR;
2027}
2028
2038int
2039nc4_rec_write_groups_types(NC_GRP_INFO_T *grp)
2040{
2041 NC_GRP_INFO_T *child_grp;
2042 NC_HDF5_GRP_INFO_T *hdf5_grp;
2043 NC_TYPE_INFO_T *type;
2044 int retval;
2045 int i;
2046
2047 assert(grp && grp->hdr.name && grp->format_grp_info);
2048 LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
2049
2050 /* Get HDF5-specific group info. */
2051 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
2052
2053 /* Create the group in the HDF5 file if it doesn't exist. */
2054 if (!hdf5_grp->hdf_grpid)
2055 if ((retval = create_group(grp)))
2056 return retval;
2057
2058 /* If this is the root group of a file with strict NC3 rules, write
2059 * an attribute. But don't leave the attribute open. */
2060 if (!grp->parent && (grp->nc4_info->cmode & NC_CLASSIC_MODEL))
2061 if ((retval = write_nc3_strict_att(hdf5_grp->hdf_grpid)))
2062 return retval;
2063
2064 /* If there are any user-defined types, write them now. */
2065 for(i=0;i<ncindexsize(grp->type);i++) {
2066 type = (NC_TYPE_INFO_T *)ncindexith(grp->type, i);
2067 assert(type);
2068 if ((retval = commit_type(grp, type)))
2069 return retval;
2070 }
2071
2072 /* If there are any child groups, write their groups and types. */
2073 for(i=0;i<ncindexsize(grp->children);i++) {
2074 if((child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children,i)) == NULL) continue;
2075 if ((retval = nc4_rec_write_groups_types(child_grp)))
2076 return retval;
2077 }
2078 return NC_NOERR;
2079}
2080
2094int
2095nc4_rec_match_dimscales(NC_GRP_INFO_T *grp)
2096{
2097 NC_GRP_INFO_T *g;
2098 NC_VAR_INFO_T *var;
2099 NC_DIM_INFO_T *dim;
2100 int retval = NC_NOERR;
2101 int i;
2102
2103 assert(grp && grp->hdr.name);
2104 LOG((4, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
2105
2106 /* Perform var dimscale match for child groups. */
2107 for (i = 0; i < ncindexsize(grp->children); i++)
2108 {
2109 g = (NC_GRP_INFO_T*)ncindexith(grp->children, i);
2110 assert(g);
2111 if ((retval = nc4_rec_match_dimscales(g)))
2112 return retval;
2113 }
2114
2115 /* Check all the vars in this group. If they have dimscale info,
2116 * try and find a dimension for them. */
2117 for (i = 0; i < ncindexsize(grp->vars); i++)
2118 {
2119 NC_HDF5_VAR_INFO_T *hdf5_var;
2120 int ndims;
2121 int d;
2122
2123 /* Get pointer to var and to the HDF5-specific var info. */
2124 var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
2125 assert(var && var->format_var_info);
2126 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
2127
2128 /* Check all vars and see if dim[i] != NULL if dimids[i] valid. */
2129 /* This loop is very odd. Under normal circumstances, var->dimid[d] is zero
2130 (from the initial calloc) which is a legitimate dimid. The code does not
2131 distinquish this case from the dimscale case where the id might actually
2132 be defined.
2133 The original nc4_find_dim searched up the group tree looking for the given
2134 dimid in one of the dim lists associated with each ancestor group.
2135 I changed nc4_fnd_dim to use the dimid directly using h5->alldims.
2136 However, here that is incorrect because it will find the dimid 0 always
2137 (if any dimensions were defined). Except that when dimscale dimids have
2138 been defined, one or more of the values in var->dimids will have a
2139 legitimate value.
2140 The solution I choose is to modify nc4_var_list_add to initialize dimids to
2141 illegal values (-1). This is another example of the problems with dimscales.
2142 */
2143 ndims = var->ndims;
2144 for (d = 0; d < ndims; d++)
2145 {
2146 if (var->dim[d] == NULL) {
2147 nc4_find_dim(grp, var->dimids[d], &var->dim[d], NULL);
2148 }
2149 /* assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d]); */
2150 }
2151
2152 /* Skip dimension scale variables */
2153 if (!hdf5_var->dimscale)
2154 {
2155 int d;
2156 int j;
2157
2158 /* Are there dimscales for this variable? */
2159 if (hdf5_var->dimscale_hdf5_objids)
2160 {
2161 for (d = 0; d < var->ndims; d++)
2162 {
2163 nc_bool_t finished = NC_FALSE;
2164 LOG((5, "%s: var %s has dimscale info...", __func__, var->hdr.name));
2165
2166 /* Check this and parent groups. */
2167 for (g = grp; g && !finished; g = g->parent)
2168 {
2169 /* Check all dims in this group. */
2170 for (j = 0; j < ncindexsize(g->dim); j++)
2171 {
2172 /* Get the HDF5 specific dim info. */
2173 NC_HDF5_DIM_INFO_T *hdf5_dim;
2174 dim = (NC_DIM_INFO_T *)ncindexith(g->dim, j);
2175 assert(dim && dim->format_dim_info);
2176 hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
2177
2178 /* Check for exact match of fileno/objid arrays
2179 * to find identical objects in HDF5 file. */
2180#if H5_VERSION_GE(1,12,0)
2181 int token_cmp;
2182 if (H5Otoken_cmp(hdf5_var->hdf_datasetid, &hdf5_var->dimscale_hdf5_objids[d].token, &hdf5_dim->hdf5_objid.token, &token_cmp) < 0)
2183 return NC_EHDFERR;
2184
2185 if (hdf5_var->dimscale_hdf5_objids[d].fileno == hdf5_dim->hdf5_objid.fileno &&
2186 token_cmp == 0)
2187#else
2188 if (hdf5_var->dimscale_hdf5_objids[d].fileno[0] == hdf5_dim->hdf5_objid.fileno[0] &&
2189 hdf5_var->dimscale_hdf5_objids[d].objno[0] == hdf5_dim->hdf5_objid.objno[0] &&
2190 hdf5_var->dimscale_hdf5_objids[d].fileno[1] == hdf5_dim->hdf5_objid.fileno[1] &&
2191 hdf5_var->dimscale_hdf5_objids[d].objno[1] == hdf5_dim->hdf5_objid.objno[1])
2192#endif
2193 {
2194 LOG((4, "%s: for dimension %d, found dim %s", __func__,
2195 d, dim->hdr.name));
2196 var->dimids[d] = dim->hdr.id;
2197 var->dim[d] = dim;
2198 finished = NC_TRUE;
2199 break;
2200 }
2201 } /* next dim */
2202 } /* next grp */
2203 LOG((5, "%s: dimid for this dimscale is %d", __func__,
2204 var->type_info->hdr.id));
2205 } /* next var->dim */
2206 }
2207 /* No dimscales for this var! Invent phony dimensions. */
2208 else
2209 {
2210 hid_t spaceid = 0;
2211 hsize_t *h5dimlen = NULL, *h5dimlenmax = NULL;
2212 int dataset_ndims;
2213
2214 /* Find the space information for this dimension. */
2215 if ((spaceid = H5Dget_space(hdf5_var->hdf_datasetid)) < 0)
2216 return NC_EHDFERR;
2217
2218 /* Get the len of each dim in the space. */
2219 if (var->ndims)
2220 {
2221 if (!(h5dimlen = malloc(var->ndims * sizeof(hsize_t))))
2222 return NC_ENOMEM;
2223 if (!(h5dimlenmax = malloc(var->ndims * sizeof(hsize_t))))
2224 {
2225 free(h5dimlen);
2226 return NC_ENOMEM;
2227 }
2228 if ((dataset_ndims = H5Sget_simple_extent_dims(spaceid, h5dimlen,
2229 h5dimlenmax)) < 0) {
2230 free(h5dimlenmax);
2231 free(h5dimlen);
2232 return NC_EHDFERR;
2233 }
2234 if (dataset_ndims != var->ndims) {
2235 free(h5dimlenmax);
2236 free(h5dimlen);
2237 return NC_EHDFERR;
2238 }
2239 }
2240 else
2241 {
2242 /* Make sure it's scalar. */
2243 if (H5Sget_simple_extent_type(spaceid) != H5S_SCALAR)
2244 return NC_EHDFERR;
2245 }
2246
2247 /* Release the space object. */
2248 if (H5Sclose(spaceid) < 0) {
2249 free(h5dimlen);
2250 free(h5dimlenmax);
2251 return NC_EHDFERR;
2252 }
2253
2254 /* Create a phony dimension for each dimension in the
2255 * dataset, unless there already is one the correct
2256 * size. */
2257 for (d = 0; d < var->ndims; d++)
2258 {
2259 int k;
2260 int match;
2261 /* Is there already a phony dimension of the correct size? */
2262 for(match=-1,k=0;k<ncindexsize(grp->dim);k++) {
2263 if((dim = (NC_DIM_INFO_T*)ncindexith(grp->dim,k)) == NULL) continue;
2264 if ((dim->len == h5dimlen[d]) &&
2265 ((h5dimlenmax[d] == H5S_UNLIMITED && dim->unlimited) ||
2266 (h5dimlenmax[d] != H5S_UNLIMITED && !dim->unlimited)))
2267 {match = k; break;}
2268 }
2269
2270 /* Didn't find a phony dim? Then create one. */
2271 if (match < 0)
2272 {
2273 char phony_dim_name[NC_MAX_NAME + 1];
2274 sprintf(phony_dim_name, "phony_dim_%d", grp->nc4_info->next_dimid);
2275 LOG((3, "%s: creating phony dim for var %s", __func__, var->hdr.name));
2276 if ((retval = nc4_dim_list_add(grp, phony_dim_name, h5dimlen[d], -1, &dim)))
2277 {
2278 free(h5dimlenmax);
2279 free(h5dimlen);
2280 return retval;
2281 }
2282 /* Create struct for HDF5-specific dim info. */
2283 if (!(dim->format_dim_info = calloc(1, sizeof(NC_HDF5_DIM_INFO_T))))
2284 return NC_ENOMEM;
2285 if (h5dimlenmax[d] == H5S_UNLIMITED)
2286 dim->unlimited = NC_TRUE;
2287 }
2288
2289 /* The variable must remember the dimid. */
2290 var->dimids[d] = dim->hdr.id;
2291 var->dim[d] = dim;
2292 } /* next dim */
2293
2294 /* Free the memory we malloced. */
2295 free(h5dimlen);
2296 free(h5dimlenmax);
2297 }
2298 }
2299 }
2300
2301 return retval;
2302}
2303
2315void
2316reportobject(int uselog, hid_t id, unsigned int type)
2317{
2318 char name[NC_HDF5_MAX_NAME];
2319 ssize_t len;
2320 const char* typename = NULL;
2321 long long printid = (long long)id;
2322
2323 len = H5Iget_name(id, name, NC_HDF5_MAX_NAME);
2324 if(len < 0) return;
2325 name[len] = '\0';
2326
2327 switch (type) {
2328 case H5F_OBJ_FILE: typename = "File"; break;
2329 case H5F_OBJ_DATASET: typename = "Dataset"; break;
2330 case H5F_OBJ_GROUP: typename = "Group"; break;
2331 case H5F_OBJ_DATATYPE: typename = "Datatype"; break;
2332 case H5F_OBJ_ATTR:
2333 typename = "Attribute";
2334 len = H5Aget_name(id, NC_HDF5_MAX_NAME, name);
2335 if(len < 0) len = 0;
2336 name[len] = '\0';
2337 break;
2338 default: typename = "<unknown>"; break;
2339 }
2340#ifdef LOGGING
2341 if(uselog) {
2342 LOG((0,"Type = %s(%lld) name='%s'",typename,printid,name));
2343 } else
2344#endif
2345 {
2346 fprintf(stderr,"Type = %s(%lld) name='%s'",typename,printid,name);
2347 }
2348
2349}
2350
2361static void
2362reportopenobjectsT(int uselog, hid_t fid, int ntypes, unsigned int* otypes)
2363{
2364 int t,i;
2365 ssize_t ocount;
2366 size_t maxobjs = -1;
2367 hid_t* idlist = NULL;
2368
2369 /* Always report somehow */
2370#ifdef LOGGING
2371 if(uselog)
2372 LOG((0,"\nReport: open objects on %lld",(long long)fid));
2373 else
2374#endif
2375 fprintf(stdout,"\nReport: open objects on %lld\n",(long long)fid);
2376 maxobjs = H5Fget_obj_count(fid,H5F_OBJ_ALL);
2377 if(idlist != NULL) free(idlist);
2378 idlist = (hid_t*)malloc(sizeof(hid_t)*maxobjs);
2379 for(t=0;t<ntypes;t++) {
2380 unsigned int ot = otypes[t];
2381 ocount = H5Fget_obj_ids(fid,ot,maxobjs,idlist);
2382 for(i=0;i<ocount;i++) {
2383 hid_t o = idlist[i];
2384 reportobject(uselog,o,ot);
2385 }
2386 }
2387 if(idlist != NULL) free(idlist);
2388}
2389
2398void
2399reportopenobjects(int uselog, hid_t fid)
2400{
2401 unsigned int OTYPES[5] = {H5F_OBJ_FILE, H5F_OBJ_DATASET, H5F_OBJ_GROUP,
2402 H5F_OBJ_DATATYPE, H5F_OBJ_ATTR};
2403
2404 reportopenobjectsT(uselog, fid ,5, OTYPES);
2405}
2406
2414void
2415showopenobjects5(NC_FILE_INFO_T* h5)
2416{
2417 NC_HDF5_FILE_INFO_T *hdf5_info;
2418
2419 assert(h5 && h5->format_file_info);
2420 hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
2421
2422 fprintf(stderr,"===== begin showopenobjects =====\n");
2423 reportopenobjects(0,hdf5_info->hdfid);
2424 fprintf(stderr,"===== end showopenobjects =====\n");
2425 fflush(stderr);
2426}
2427
2436void
2437showopenobjects(int ncid)
2438{
2439 NC_FILE_INFO_T* h5 = NULL;
2440
2441 /* Find our metadata for this file. */
2442 if (nc4_find_nc_grp_h5(ncid, NULL, NULL, &h5) != NC_NOERR)
2443 fprintf(stderr,"failed\n");
2444 else
2445 showopenobjects5(h5);
2446 fflush(stderr);
2447}
2448
2460int
2461NC4_hdf5get_libversion(unsigned* major,unsigned* minor,unsigned* release)
2462{
2463 if(H5get_libversion(major,minor,release) < 0)
2464 return NC_EHDFERR;
2465 return NC_NOERR;
2466}
2467
2478int
2479NC4_hdf5get_superblock(struct NC_FILE_INFO* h5, int* idp)
2480{
2481 NC_HDF5_FILE_INFO_T *hdf5_info;
2482 int stat = NC_NOERR;
2483 unsigned super;
2484 hid_t plist = -1;
2485
2486 assert(h5 && h5->format_file_info);
2487 hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
2488
2489 if((plist = H5Fget_create_plist(hdf5_info->hdfid)) < 0)
2490 {stat = NC_EHDFERR; goto done;}
2491 if(H5Pget_version(plist, &super, NULL, NULL, NULL) < 0)
2492 {stat = NC_EHDFERR; goto done;}
2493 if(idp) *idp = (int)super;
2494done:
2495 if(plist >= 0) H5Pclose(plist);
2496 return stat;
2497}
2498
2499static int NC4_strict_att_exists(NC_FILE_INFO_T*);
2500static int NC4_walk(hid_t, int*);
2501
2527int
2528NC4_isnetcdf4(struct NC_FILE_INFO* h5)
2529{
2530 int stat;
2531 int isnc4 = 0;
2532 int exists;
2533 int count;
2534
2535 /* Look for NC3_STRICT_ATT_NAME */
2536 exists = NC4_strict_att_exists(h5);
2537 if(exists)
2538 goto done;
2539 /* attribute did not exist */
2540 /* => last resort: walk the HDF5 file looking for markers */
2541 count = 0;
2542 stat = NC4_walk(((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid,
2543 &count);
2544 if(stat != NC_NOERR)
2545 isnc4 = 0;
2546 else /* Threshold is at least two matches */
2547 isnc4 = (count >= 2);
2548
2549done:
2550 return isnc4;
2551}
2552
2561static int
2562NC4_strict_att_exists(NC_FILE_INFO_T *h5)
2563{
2564 hid_t grpid = -1;
2565 htri_t attr_exists;
2566
2567 /* Get root group ID. */
2568 grpid = ((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid;
2569
2570 /* See if the NC3_STRICT_ATT_NAME attribute exists */
2571 if ((attr_exists = H5Aexists(grpid, NC3_STRICT_ATT_NAME)) < 0)
2572 return 1;
2573 return (attr_exists?1:0);
2574}
2575
2585static int
2586NC4_walk(hid_t gid, int* countp)
2587{
2588 int ncstat = NC_NOERR;
2589 int i,j,na;
2590 ssize_t len;
2591 hsize_t nobj;
2592 herr_t err;
2593 int otype;
2594 hid_t grpid, dsid;
2595 char name[NC_HDF5_MAX_NAME];
2596
2597 /* walk group members of interest */
2598 err = H5Gget_num_objs(gid, &nobj);
2599 if(err < 0) return err;
2600
2601 for(i = 0; i < nobj; i++) {
2602 /* Get name & kind of object in the group */
2603 len = H5Gget_objname_by_idx(gid,(hsize_t)i,name,(size_t)NC_HDF5_MAX_NAME);
2604 if(len < 0) return len;
2605
2606 otype = H5Gget_objtype_by_idx(gid,(size_t)i);
2607 switch(otype) {
2608 case H5G_GROUP:
2609 grpid = H5Gopen1(gid,name);
2610 NC4_walk(grpid,countp);
2611 H5Gclose(grpid);
2612 break;
2613 case H5G_DATASET: /* variables */
2614 /* Check for phony_dim */
2615 if(strcmp(name,"phony_dim")==0)
2616 *countp = *countp + 1;
2617 dsid = H5Dopen1(gid,name);
2618 na = H5Aget_num_attrs(dsid);
2619 for(j = 0; j < na; j++) {
2620 hid_t aid = H5Aopen_idx(dsid,(unsigned int) j);
2621 if(aid >= 0) {
2622 const NC_reservedatt* ra;
2623 ssize_t len = H5Aget_name(aid, NC_HDF5_MAX_NAME, name);
2624 if(len < 0) return len;
2625 /* Is this a netcdf-4 marker attribute */
2626 /* Is this a netcdf-4 marker attribute */
2627 ra = NC_findreserved(name);
2628 if(ra != NULL)
2629 *countp = *countp + 1;
2630 }
2631 H5Aclose(aid);
2632 }
2633 H5Dclose(dsid);
2634 break;
2635 default:/* ignore */
2636 break;
2637 }
2638 }
2639 return ncstat;
2640}
2641
EXTERNL int nc_free_vlen(nc_vlen_t *vl)
Free memory in a VLEN object.
Definition: dvlen.c:43
Main header file for the C API.
#define NC_EBADTYPE
Not a netcdf data type.
Definition: netcdf.h:410
#define NC_UINT
unsigned 4-byte int
Definition: netcdf.h:44
#define NC_ENDIAN_NATIVE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:294
#define NC_EFILTER
Filter operation failed.
Definition: netcdf.h:513
#define NC_QUANTIZE_GRANULARBR_ATT_NAME
When quantization is used for a variable, an attribute of the appropriate name is added.
Definition: netcdf.h:344
#define NC_INT
signed 4 byte integer
Definition: netcdf.h:38
#define NC_ENDIAN_BIG
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:296
#define NC_MAX_VAR_DIMS
max per variable dimensions
Definition: netcdf.h:282
#define NC_BYTE
signed 1 byte integer
Definition: netcdf.h:35
#define NC_EPERM
Write to read only.
Definition: netcdf.h:379
#define NC_QUANTIZE_BITROUND
Use BitRound quantization.
Definition: netcdf.h:338
#define NC_EDIMSCALE
Problem with HDF5 dimscales.
Definition: netcdf.h:504
#define NC_VLEN
vlen (variable-length) types
Definition: netcdf.h:53
#define NC_NAT
Not A Type.
Definition: netcdf.h:34
#define NC_DOUBLE
double precision floating point number
Definition: netcdf.h:41
#define NC_UBYTE
unsigned 1 byte int
Definition: netcdf.h:42
#define NC_FLOAT
single precision floating point number
Definition: netcdf.h:40
#define NC_ENOMEM
Memory allocation (malloc) failure.
Definition: netcdf.h:448
#define NC_COMPOUND
compound types
Definition: netcdf.h:56
#define NC_CHUNKED
In HDF5 files you can set storage for each variable to be either contiguous or chunked,...
Definition: netcdf.h:304
#define NC_SHORT
signed 2 byte integer
Definition: netcdf.h:37
#define NC_QUANTIZE_GRANULARBR
Use Granular BitRound quantization.
Definition: netcdf.h:337
#define NC_QUANTIZE_BITGROOM
Use BitGroom quantization.
Definition: netcdf.h:336
#define NC_ENUM
enum types
Definition: netcdf.h:55
#define NC_INT64
signed 8-byte int
Definition: netcdf.h:45
#define NC_EATTMETA
Problem with attribute metadata.
Definition: netcdf.h:487
#define NC_EHDFERR
Error at HDF5 layer.
Definition: netcdf.h:481
#define NC_CONTIGUOUS
In HDF5 files you can set storage for each variable to be either contiguous or chunked,...
Definition: netcdf.h:305
#define NC_GLOBAL
Attribute id to put/get a global attribute.
Definition: netcdf.h:254
#define NC_UINT64
unsigned 8-byte int
Definition: netcdf.h:46
#define NC_COMPACT
In HDF5 files you can set storage for each variable to be either contiguous or chunked,...
Definition: netcdf.h:306
#define NC_EFILEMETA
Problem with file metadata.
Definition: netcdf.h:485
#define NC_ENOTVAR
Variable not found.
Definition: netcdf.h:422
#define NC_EINVAL
Invalid Argument.
Definition: netcdf.h:378
#define NC_CLASSIC_MODEL
Enforce classic model on netCDF-4.
Definition: netcdf.h:140
#define NC_ENDIAN_LITTLE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:295
#define NC_MAX_NAME
Maximum for classic library.
Definition: netcdf.h:281
#define NC_NOERR
No Error.
Definition: netcdf.h:368
#define NC_EDIMMETA
Problem with dimension metadata.
Definition: netcdf.h:486
#define NC_USHORT
unsigned 2-byte int
Definition: netcdf.h:43
#define NC_OPAQUE
opaque types
Definition: netcdf.h:54
#define NC_STRING
string
Definition: netcdf.h:47
#define NC_QUANTIZE_BITGROOM_ATT_NAME
When quantization is used for a variable, an attribute of the appropriate name is added.
Definition: netcdf.h:343
#define NC_CHAR
ISO/ASCII character.
Definition: netcdf.h:36
#define NC_EVARMETA
Problem with variable metadata.
Definition: netcdf.h:488
#define NC_QUANTIZE_BITROUND_ATT_NAME
When quantization is used for a variable, an attribute of the appropriate name is added.
Definition: netcdf.h:345
int nc_type
The nc_type type is just an int.
Definition: netcdf.h:25
This is the type of arrays of vlens.
Definition: netcdf.h:746