39 #include <sys/types.h>
45 #include <D4BaseTypeFactory.h>
46 #include <BESDMRResponse.h>
47 #include <ObjMemCache.h>
50 #include <mime_util.h>
55 #include <BESDASResponse.h>
56 #include <BESDDSResponse.h>
57 #include <BESDataDDSResponse.h>
58 #include <Ancillary.h>
60 #include <BESDapNames.h>
61 #include <BESResponseNames.h>
62 #include <BESContainer.h>
63 #include <BESResponseHandler.h>
64 #include <BESVersionInfo.h>
65 #include <BESServiceRegistry.h>
67 #include <BESDapError.h>
68 #include <BESInternalFatalError.h>
69 #include <TheBESKeys.h>
72 #include "config_hdf5.h"
74 #define HDF5_NAME "h5"
81 void get_attr_contents(AttrTable* temp_table);
85 void write_das_to_file(DAS*das_ptr,FILE* das_file);
88 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file);
91 void write_container_name_to_file(
const string&,FILE* das_file);
94 void write_das_attr_info(AttrTable* dtp,
const string&,
const string&,FILE * das_file);
97 char* copy_str(
char *temp_ptr,
const string & str);
101 char* obtain_str(
char*temp_ptr,
string & str);
104 char* get_attr_info_from_dc(
char*temp_pointer,DAS *das,AttrTable *at);
107 bool check_beskeys(
const string);
110 static unsigned int get_uint_key(
const string &key,
unsigned int def_val);
111 static unsigned long get_ulong_key(
const string &key,
unsigned long def_val);
114 static float get_float_key(
const string &key,
float def_val);
117 static string get_beskeys(
const string&);
120 extern void read_cfdas(DAS &das,
const string & filename,hid_t fileid);
121 extern void read_cfdds(DDS &dds,
const string & filename,hid_t fileid);
125 unsigned int HDF5RequestHandler::_mdcache_entries = 500;
126 unsigned int HDF5RequestHandler::_lrdcache_entries = 0;
127 unsigned int HDF5RequestHandler::_srdcache_entries = 0;
128 float HDF5RequestHandler::_cache_purge_level = 0.2;
133 ObjMemCache *HDF5RequestHandler::datadds_cache = 0;
136 ObjMemCache *HDF5RequestHandler::lrdata_mem_cache = 0;
137 ObjMemCache *HDF5RequestHandler::srdata_mem_cache = 0;
140 bool HDF5RequestHandler::_usecf =
false;
141 bool HDF5RequestHandler::_pass_fileid =
false;
142 bool HDF5RequestHandler::_disable_structmeta =
false;
143 bool HDF5RequestHandler::_disable_ecsmeta =
false;
144 bool HDF5RequestHandler::_keep_var_leading_underscore =
false;
145 bool HDF5RequestHandler::_check_name_clashing =
false;
146 bool HDF5RequestHandler::_add_path_attrs =
false;
147 bool HDF5RequestHandler::_drop_long_string =
false;
148 bool HDF5RequestHandler::_fillvalue_check =
false;
149 bool HDF5RequestHandler::_check_ignore_obj =
false;
150 bool HDF5RequestHandler::_flatten_coor_attr =
false;
151 bool HDF5RequestHandler::_default_handle_dimension =
false;
152 bool HDF5RequestHandler::_eos5_rm_convention_attr_path =
true;
154 bool HDF5RequestHandler::_common_cache_dirs =
false;
156 bool HDF5RequestHandler::_use_disk_cache =
false;
157 bool HDF5RequestHandler::_use_disk_dds_cache =
false;
158 string HDF5RequestHandler::_disk_cache_dir =
"";
159 string HDF5RequestHandler::_disk_cachefile_prefix =
"";
160 unsigned long long HDF5RequestHandler::_disk_cache_size =0;
163 bool HDF5RequestHandler::_disk_cache_comp_data =
false;
164 bool HDF5RequestHandler::_disk_cache_float_only_comp_data =
false;
165 float HDF5RequestHandler::_disk_cache_comp_threshold =1.0;
166 unsigned long HDF5RequestHandler::_disk_cache_var_size =0;
168 bool HDF5RequestHandler::_use_disk_meta_cache =
false;
169 string HDF5RequestHandler::_disk_meta_cache_path =
"";
171 bool HDF5RequestHandler::_use_latlon_disk_cache =
false;
172 long HDF5RequestHandler::_latlon_disk_cache_size =0;
173 string HDF5RequestHandler::_latlon_disk_cache_dir =
"";
174 string HDF5RequestHandler::_latlon_disk_cachefile_prefix=
"";
176 DMR* HDF5RequestHandler::dmr_int64 = 0;
183 string HDF5RequestHandler::_stp_east_filename;
184 string HDF5RequestHandler::_stp_north_filename;
185 vector<string> HDF5RequestHandler::lrd_cache_dir_list;
186 vector<string> HDF5RequestHandler::lrd_non_cache_dir_list;
187 vector<string> HDF5RequestHandler::lrd_var_cache_file_list;
194 HDF5RequestHandler::HDF5RequestHandler(
const string & name)
198 BESDEBUG(HDF5_NAME,
"In HDF5RequestHandler::HDF5RequestHandler" << endl);
200 add_handler(DAS_RESPONSE, HDF5RequestHandler::hdf5_build_das);
201 add_handler(DDS_RESPONSE, HDF5RequestHandler::hdf5_build_dds);
202 add_handler(DATA_RESPONSE, HDF5RequestHandler::hdf5_build_data);
203 add_handler(DMR_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
204 add_handler(DAP4DATA_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
206 add_handler(HELP_RESPONSE, HDF5RequestHandler::hdf5_build_help);
207 add_handler(VERS_RESPONSE, HDF5RequestHandler::hdf5_build_version);
210 HDF5RequestHandler::_mdcache_entries = get_uint_key(
"H5.MetaDataMemCacheEntries", 0);
211 HDF5RequestHandler::_lrdcache_entries = get_uint_key(
"H5.LargeDataMemCacheEntries", 0);
212 HDF5RequestHandler::_srdcache_entries = get_uint_key(
"H5.SmallDataMemCacheEntries", 0);
213 HDF5RequestHandler::_cache_purge_level = get_float_key(
"H5.CachePurgeLevel", 0.2);
215 if (get_mdcache_entries()) {
216 das_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
217 dds_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
218 datadds_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
219 dmr_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
223 _usecf = check_beskeys(
"H5.EnableCF");
226 _pass_fileid = check_beskeys(
"H5.EnablePassFileID");
227 _disable_structmeta = check_beskeys(
"H5.DisableStructMetaAttr");
228 _disable_ecsmeta = check_beskeys(
"H5.DisableECSMetaAttr");
229 _keep_var_leading_underscore = check_beskeys(
"H5.KeepVarLeadingUnderscore");
230 _check_name_clashing = check_beskeys(
"H5.EnableCheckNameClashing");
231 _add_path_attrs = check_beskeys(
"H5.EnableAddPathAttrs");
232 _drop_long_string = check_beskeys(
"H5.EnableDropLongString");
233 _fillvalue_check = check_beskeys(
"H5.EnableFillValueCheck");
234 _check_ignore_obj = check_beskeys(
"H5.CheckIgnoreObj");
235 _flatten_coor_attr = check_beskeys(
"H5.ForceFlattenNDCoorAttr");
236 _default_handle_dimension = check_beskeys(
"H5.DefaultHandleDimension");
237 _eos5_rm_convention_attr_path= check_beskeys(
"H5.RmConventionAttrPath");
239 _use_disk_cache = check_beskeys(
"H5.EnableDiskDataCache");
240 _disk_cache_dir = get_beskeys(
"H5.DiskCacheDataPath");
241 _disk_cachefile_prefix = get_beskeys(
"H5.DiskCacheFilePrefix");
242 _disk_cache_size = get_ulong_key(
"H5.DiskCacheSize",0);
244 _disk_cache_comp_data = check_beskeys(
"H5.DiskCacheComp");
245 _disk_cache_float_only_comp_data = check_beskeys(
"H5.DiskCacheFloatOnlyComp");
246 _disk_cache_comp_threshold = get_float_key(
"H5.DiskCacheCompThreshold",1.0);
247 _disk_cache_var_size = 1024*get_uint_key(
"H5.DiskCacheCompVarSize",0);
249 _use_disk_meta_cache = check_beskeys(
"H5.EnableDiskMetaDataCache");
250 _use_disk_dds_cache = check_beskeys(
"H5.EnableDiskDDSCache");
251 _disk_meta_cache_path = get_beskeys(
"H5.DiskMetaDataCachePath");
253 _use_latlon_disk_cache = check_beskeys(
"H5.EnableEOSGeoCacheFile");
254 _latlon_disk_cache_size = get_uint_key(
"H5.Cache.latlon.size",0);
255 _latlon_disk_cache_dir = get_beskeys(
"H5.Cache.latlon.path");
256 _latlon_disk_cachefile_prefix= get_beskeys(
"H5.Cache.latlon.prefix");
259 if(get_lrdcache_entries()) {
260 lrdata_mem_cache =
new ObjMemCache(get_lrdcache_entries(), get_cache_purge_level());
261 if(
true == check_beskeys(
"H5.LargeDataMemCacheConfig")) {
262 _common_cache_dirs =obtain_lrd_common_cache_dirs();
264 if(
false == _common_cache_dirs)
265 cerr<<
"No specific cache info"<<endl;
270 if(get_srdcache_entries()) {
272 BESDEBUG(HDF5_NAME,
"Generate memory cache for smaller coordinate variables" << endl);
273 srdata_mem_cache =
new ObjMemCache(get_srdcache_entries(),get_cache_purge_level());
277 if(_disk_cache_comp_data ==
true && _use_disk_cache ==
true) {
278 if(_disk_cache_comp_threshold < 1.0) {
280 ss<< _disk_cache_comp_threshold;
281 string _comp_threshold_str(ss.str());
282 string invalid_comp_threshold =
"The Compression Threshold is the total size of the variable array";
283 invalid_comp_threshold+=
" divided by the storage size of compressed array. It should always be >1";
284 invalid_comp_threshold+=
" The current threhold set at h5.conf is ";
285 invalid_comp_threshold+=_comp_threshold_str;
286 invalid_comp_threshold+=
" . Go back to h5.conf and change the H5.DiskCacheCompThreshold to a >1.0 number.";
290 _stp_east_filename = get_beskeys(
"H5.STPEastFileName");
291 _stp_north_filename = get_beskeys(
"H5.STPNorthFileName");
295 BESDEBUG(HDF5_NAME,
"Exiting HDF5RequestHandler::HDF5RequestHandler" << endl);
298 HDF5RequestHandler::~HDF5RequestHandler()
303 delete datadds_cache;
305 delete lrdata_mem_cache;
306 delete srdata_mem_cache;
315 hid_t cf_fileid = -1;
330 DAS *das = bdas->get_das();
333 DAS *cached_das_ptr = 0;
334 bool use_das_cache =
false;
336 cached_das_ptr =
static_cast<DAS*
>(das_cache->
get(filename));
338 use_das_cache =
true;
340 if (
true == use_das_cache) {
343 BESDEBUG(HDF5_NAME,
"DAS Cached hit for : " << filename << endl);
344 *das = *cached_das_ptr;
348 bool das_from_dc =
false;
349 string das_cache_fname;
352 if(_use_disk_meta_cache ==
true) {
354 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
355 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
357 if(access(das_cache_fname.c_str(),F_OK) !=-1)
363 if(
true == das_from_dc) {
364 read_das_from_disk_cache(das_cache_fname,das);
369 BESDEBUG(HDF5_NAME,
"HDF5 DAS reading DAS from the disk cache. For memory cache, DAS added to the cache for : " << filename << endl);
370 das_cache->
add(
new DAS(*das), filename);
375 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
376 if (
true == _usecf) {
378 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
380 string invalid_file_msg=
"Could not open this HDF5 file ";
381 invalid_file_msg +=filename;
382 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
383 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
384 invalid_file_msg +=
" distributor.";
388 if(HDF5RequestHandler::get_dmr_64bit_int()!=NULL)
389 HDF5RequestHandler::set_dmr_64bit_int(NULL);
390 read_cfdas( *das,filename,cf_fileid);
396 string invalid_file_msg=
"Could not open this HDF5 file ";
397 invalid_file_msg +=filename;
398 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
399 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
400 invalid_file_msg +=
" distributor.";
410 Ancillary::read_ancillary_das( *das, filename ) ;
414 AttrTable* top_table = das->get_top_level_attributes();
415 get_attr_contents(top_table);
418 AttrTable::Attr_iter start_aiter=das->var_begin();
419 AttrTable::Attr_iter it = start_aiter;
420 AttrTable::Attr_iter end_aiter = das->var_end();
421 while(it != end_aiter) {
422 AttrTable* temp_table = das->get_table(it);
424 cerr<<
"var_begin"<<endl;
425 temp_table->print(cerr);
433 BESDEBUG(HDF5_NAME,
"DAS added to the cache for : " << filename << endl);
434 das_cache->
add(
new DAS(*das), filename);
439 if(das_cache_fname!=
"") {
440 BESDEBUG(HDF5_NAME,
"HDF5 Build DAS: Write DAS to disk cache " << das_cache_fname << endl);
441 write_das_to_disk_cache(das_cache_fname,das);
448 catch(InternalErr & e) {
452 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
459 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
466 string s =
"unknown exception caught building HDF5 DAS";
476 void HDF5RequestHandler::get_dds_with_attributes(
BESDDSResponse*bdds,
BESDataDDSResponse*data_bdds,
const string &container_name,
const string& filename,
const string &dds_cache_fname,
const string &das_cache_fname,
bool dds_from_dc,
bool das_from_dc,
bool build_data)
479 if(
true == build_data)
480 dds = data_bdds->get_dds();
485 hid_t cf_fileid = -1;
490 DDS* cached_dds_ptr = 0;
491 bool use_dds_cache =
false;
493 cached_dds_ptr =
static_cast<DDS*
>(dds_cache->
get(filename));
495 use_dds_cache =
true;
496 if (
true == use_dds_cache) {
499 BESDEBUG(HDF5_NAME,
"DDS Metadata Cached hit for : " << filename << endl);
500 *dds = *cached_dds_ptr;
502 else if (
true ==dds_from_dc) {
503 read_dds_from_disk_cache(bdds,data_bdds,build_data,container_name,filename,dds_cache_fname,das_cache_fname,-1,das_from_dc);
506 BESDEBUG(HDF5_NAME,
"Build DDS from the HDF5 file. " << filename << endl);
507 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
508 dds->filename(filename);
513 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
515 string invalid_file_msg=
"Could not open this HDF5 file ";
516 invalid_file_msg +=filename;
517 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
518 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
519 invalid_file_msg +=
" distributor.";
524 if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
525 HDF5RequestHandler::set_dmr_64bit_int(NULL);
526 read_cfdds(*dds,filename,cf_fileid);
532 string invalid_file_msg=
"Could not open this HDF5 file ";
533 invalid_file_msg +=filename;
534 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
535 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
536 invalid_file_msg +=
" distributor.";
540 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
544 if (!dds->check_semantics()) {
546 throw InternalErr(__FILE__, __LINE__,
547 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
550 Ancillary::read_ancillary_dds( *dds, filename ) ;
553 if(dds_cache_fname!=
"" && dds_from_dc ==
false)
554 write_dds_to_disk_cache(dds_cache_fname,dds);
563 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
569 BESDEBUG(HDF5_NAME,
"DDS added to the cache for : " << filename << endl);
570 dds_cache->
add(
new DDS(*dds), filename);
581 catch(InternalErr & e) {
589 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
599 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
609 string s =
"unknown exception caught building HDF5 DDS";
615 void HDF5RequestHandler::get_dds_without_attributes_datadds(
BESDataDDSResponse*data_bdds,
const string &container_name,
const string& filename)
617 DDS *dds = data_bdds->get_dds();
621 hid_t cf_fileid = -1;
626 DDS* cached_dds_ptr = 0;
627 bool use_datadds_cache =
false;
629 cached_dds_ptr =
static_cast<DDS*
>(datadds_cache->
get(filename));
631 use_datadds_cache =
true;
632 if (
true == use_datadds_cache) {
635 BESDEBUG(HDF5_NAME,
"DataDDS Metadata Cached hit for : " << filename << endl);
636 *dds = *cached_dds_ptr;
639 BESDEBUG(HDF5_NAME,
"Build DDS from the HDF5 file. " << filename << endl);
640 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
641 dds->filename(filename);
646 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
648 string invalid_file_msg=
"Could not open this HDF5 file ";
649 invalid_file_msg +=filename;
650 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
651 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
652 invalid_file_msg +=
" distributor.";
657 if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
658 HDF5RequestHandler::set_dmr_64bit_int(NULL);
659 read_cfdds(*dds,filename,cf_fileid);
665 string invalid_file_msg=
"Could not open this HDF5 file ";
666 invalid_file_msg +=filename;
667 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
668 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
669 invalid_file_msg +=
" distributor.";
673 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
677 if (!dds->check_semantics()) {
679 throw InternalErr(__FILE__, __LINE__,
680 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
683 Ancillary::read_ancillary_dds( *dds, filename ) ;
693 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
700 BESDEBUG(HDF5_NAME,
"DataDDS added to the cache for : " << filename << endl);
701 datadds_cache->
add(
new DDS(*dds), filename);
710 BESDEBUG(HDF5_NAME,
"Data ACCESS build_data(): set the including attribute flag to false: "<<filename << endl);
711 data_bdds->set_ia_flag(
false);
714 catch(InternalErr & e) {
722 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
732 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
742 string s =
"unknown exception caught building HDF5 DDS";
752 void HDF5RequestHandler::get_dds_with_attributes(
const string &filename,
const string &container_name, DDS*dds) {
756 hid_t cf_fileid = -1;
761 DDS* cached_dds_ptr = 0;
762 if (dds_cache && (cached_dds_ptr =
static_cast<DDS*
>(dds_cache->
get(filename)))) {
765 BESDEBUG(HDF5_NAME,
"DDS Cached hit for : " << filename << endl);
766 *dds = *cached_dds_ptr;
771 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
772 if (!container_name.empty())
773 dds->container_name(container_name);
774 dds->filename(filename);
780 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
781 string dds_filename =
"/tmp/"+base_filename+
"_dds";
782 FILE *dds_file = fopen(dds_filename.c_str(),
"r");
783 cerr<<
"before parsing "<<endl;
785 DDS tdds(&tf,name_path(filename),
"3.2");
786 tdds.filename(filename);
788 tdds.parse(dds_file);
790 cache_dds =
new DDS(tdds);
796 cerr<<
"after parsing "<<endl;
805 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
807 string invalid_file_msg=
"Could not open this HDF5 file ";
808 invalid_file_msg +=filename;
809 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
810 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
811 invalid_file_msg +=
" distributor.";
815 read_cfdds(*dds,filename,cf_fileid);
818 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
819 string dds_filename =
"/tmp/"+base_filename+
"_dds";
820 FILE *dds_file = fopen(dds_filename.c_str(),
"w");
821 dds->print(dds_file);
829 string invalid_file_msg=
"Could not open this HDF5 file ";
830 invalid_file_msg +=filename;
831 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
832 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
833 invalid_file_msg +=
" distributor.";
837 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
843 if (!dds->check_semantics()) {
845 throw InternalErr(__FILE__, __LINE__,
846 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
849 Ancillary::read_ancillary_dds( *dds, filename ) ;
855 if (das_cache && (das =
static_cast<DAS*
>(das_cache->
get(filename)))) {
856 BESDEBUG(HDF5_NAME,
"DAS Cached hit for : " << filename << endl);
857 dds->transfer_attributes(das);
864 if (!container_name.empty())
865 das->container_name(container_name);
867 if (
true == _usecf) {
870 read_cfdas( *das,filename,cf_fileid);
883 Ancillary::read_ancillary_das( *das, filename ) ;
885 dds->transfer_attributes(das);
891 BESDEBUG(HDF5_NAME,
"DAS added to the cache for : " << filename << endl);
893 das_cache->
add(das, filename);
902 BESDEBUG(HDF5_NAME,
"DDS added to the cache for : " << filename << endl);
903 dds_cache->
add(
new DDS(*dds), filename);
911 catch(InternalErr & e) {
919 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
929 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
939 string s =
"unknown exception caught building HDF5 DDS";
962 bool dds_from_dc =
false;
963 bool das_from_dc =
false;
964 bool build_data =
false;
965 string dds_cache_fname;
966 string das_cache_fname;
968 if(_use_disk_meta_cache ==
true) {
970 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
973 if(_use_disk_dds_cache ==
true) {
974 dds_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_dds";
975 if(access(dds_cache_fname.c_str(),F_OK) !=-1)
979 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
981 if(access(das_cache_fname.c_str(),F_OK) !=-1)
986 get_dds_with_attributes(bdds, NULL,container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
990 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
991 string dds_filename =
"/tmp/"+base_filename+
"_dds";
994 DDS tdds(&tf,name_path(filename),
"3.2");
995 tdds.filename(filename);
998 FILE *dds_file = fopen(dds_filename.c_str(),
"r");
999 tdds.parse(dds_file);
1001 DDS* cache_dds =
new DDS(tdds);
1012 catch(InternalErr & e) {
1014 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1015 __FILE__, __LINE__);
1019 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1020 __FILE__, __LINE__);
1024 string s =
"unknown exception caught building HDF5 DDS";
1036 if(
true == _pass_fileid)
1037 return hdf5_build_data_with_IDs(dhi);
1051 bool dds_from_dc =
false;
1052 bool das_from_dc =
false;
1053 bool build_data =
true;
1054 string dds_cache_fname;
1055 string das_cache_fname;
1059 if(_use_disk_meta_cache ==
true) {
1061 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1062 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
1064 if(access(das_cache_fname.c_str(),F_OK) !=-1)
1070 get_dds_without_attributes_datadds(bdds,container_name,filename);
1076 catch(InternalErr & e) {
1078 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1079 __FILE__, __LINE__);
1083 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1084 __FILE__, __LINE__);
1088 string s =
"unknown exception caught building HDF5 DDS";
1099 BESDEBUG(
"h5",
"Building DataDDS by passing file IDs. "<<endl);
1100 hid_t cf_fileid = -1;
1104 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1105 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1107 string invalid_file_msg=
"Could not open this HDF5 file ";
1108 invalid_file_msg +=filename;
1109 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1110 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1111 invalid_file_msg +=
" distributor.";
1125 delete bdds->get_dds();
1129 hdds->setHDF5Dataset(cf_fileid);
1131 read_cfdds( *hdds,filename,cf_fileid);
1133 if (!hdds->check_semantics()) {
1135 throw InternalErr(__FILE__, __LINE__,
1136 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1139 Ancillary::read_ancillary_dds( *hdds, filename ) ;
1141 DAS *das =
new DAS ;
1144 read_cfdas( *das,filename,cf_fileid);
1145 Ancillary::read_ancillary_das( *das, filename ) ;
1147 hdds->transfer_attributes(das);
1153 catch(InternalErr & e) {
1155 H5Fclose(cf_fileid);
1156 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1157 __FILE__, __LINE__);
1161 H5Fclose(cf_fileid);
1162 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1163 __FILE__, __LINE__);
1167 H5Fclose(cf_fileid);
1168 string s =
"unknown exception caught building HDF5 DataDDS";
1186 DMR *dmr = bes_dmr.get_dmr();
1190 hid_t cf_fileid = -1;
1194 DMR* cached_dmr_ptr = 0;
1195 bool use_dmr_cache =
false;
1197 cached_dmr_ptr =
static_cast<DMR*
>(dmr_cache->
get(filename));
1199 use_dmr_cache =
true;
1200 if (
true == use_dmr_cache) {
1202 BESDEBUG(HDF5_NAME,
"DMR Cached hit for : " << filename << endl);
1203 *dmr = *cached_dmr_ptr;
1207 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1208 D4BaseTypeFactory MyD4TypeFactory;
1209 dmr->set_factory(&MyD4TypeFactory);
1213 if(
true == _pass_fileid)
1214 return hdf5_build_dmr_with_IDs(dhi);
1216 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1218 string invalid_file_msg=
"Could not open this HDF5 file ";
1219 invalid_file_msg +=filename;
1220 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1221 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1222 invalid_file_msg +=
" distributor.";
1226 BaseTypeFactory factory;
1227 DDS dds(&factory, name_path(filename),
"3.2");
1228 dds.filename(filename);
1234 HDF5RequestHandler::set_dmr_64bit_int(dmr);
1235 read_cfdds( dds,filename,cf_fileid);
1236 if (!dds.check_semantics()) {
1238 throw InternalErr(__FILE__, __LINE__,
1239 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1242 read_cfdas(das,filename,cf_fileid);
1243 Ancillary::read_ancillary_das( das, filename ) ;
1245 dds.transfer_attributes(&das);
1249 H5Fclose(cf_fileid);
1251 dmr->build_using_dds(dds);
1259 string invalid_file_msg=
"Could not open this HDF5 file ";
1260 invalid_file_msg +=filename;
1261 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1262 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1263 invalid_file_msg +=
" distributor.";
1267 bool use_dimscale =
false;
1268 if(
true == _default_handle_dimension)
1269 use_dimscale = check_dimscale(fileid);
1270 dmr->set_name(name_path(filename));
1271 dmr->set_filename(name_path(filename));
1273 D4Group* root_grp = dmr->root();
1274 breadth_first(fileid,(
char*)
"/",root_grp,filename.c_str(),use_dimscale);
1277 if(
true == use_dimscale)
1279 breadth_first(fileid,(
char*)
"/",root_grp,filename.c_str(),
true);
1281 depth_first(fileid,(
char*)
"/",root_grp,filename.c_str());
1292 BESDEBUG(HDF5_NAME,
"DMR added to the cache for : " << filename << endl);
1293 dmr_cache->
add(
new DMR(*dmr), filename);
1298 catch(InternalErr & e) {
1301 H5Fclose(cf_fileid);
1305 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1306 __FILE__, __LINE__);
1311 H5Fclose(cf_fileid);
1314 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1315 __FILE__, __LINE__);
1320 H5Fclose(cf_fileid);
1323 string s =
"unknown exception caught building HDF5 DMR";
1336 dmr->set_factory(0);
1345 BESDEBUG(
"h5",
"Building DMR with passing file IDs. "<<endl);
1347 hid_t cf_fileid = -1;
1349 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1350 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1352 string invalid_file_msg=
"Could not open this HDF5 file ";
1353 invalid_file_msg +=filename;
1354 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1355 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1356 invalid_file_msg +=
" distributor.";
1360 BaseTypeFactory factory;
1361 DDS dds(&factory, name_path(filename),
"3.2");
1362 dds.filename(filename);
1370 read_cfdds( dds,filename,cf_fileid);
1372 if (!dds.check_semantics()) {
1374 throw InternalErr(__FILE__, __LINE__,
1375 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1378 Ancillary::read_ancillary_dds( dds, filename ) ;
1381 read_cfdas(das,filename,cf_fileid);
1383 Ancillary::read_ancillary_das( das, filename ) ;
1385 dds.transfer_attributes(&das);
1392 catch(InternalErr & e) {
1395 H5Fclose(cf_fileid);
1397 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1398 __FILE__, __LINE__);
1403 H5Fclose(cf_fileid);
1405 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1406 __FILE__, __LINE__);
1411 H5Fclose(cf_fileid);
1413 string s =
"unknown exception caught building HDF5 DataDDS";
1427 DMR *dmr = bes_dmr.get_dmr();
1428 D4BaseTypeFactory MyD4TypeFactory;
1429 dmr->set_factory(&MyD4TypeFactory);
1430 dmr->build_using_dds(dds);
1433 hdf5_dmr->setHDF5Dataset(cf_fileid);
1435 bes_dmr.set_dmr(hdf5_dmr);
1444 hdf5_dmr->set_factory(0);
1456 string add_info=
"Just for Test";
1458 map<string,string> attrs ;
1459 attrs[
"name"] = MODULE_NAME ;
1460 attrs[
"version"] = MODULE_VERSION ;
1461 list<string> services ;
1462 BESServiceRegistry::TheRegistry()->
services_handled( HDF5_NAME, services );
1463 if( services.size() > 0 )
1466 attrs[
"handles"] = handles ;
1468 info->begin_tag(
"module", &attrs ) ;
1469 info->end_tag(
"module" ) ;
1482 info->add_module( MODULE_NAME, MODULE_VERSION ) ;
1488 bool HDF5RequestHandler::obtain_lrd_common_cache_dirs()
1490 string lrd_config_fpath;
1491 string lrd_config_fname;
1494 lrd_config_fpath = get_beskeys(
"H5.DataCachePath");
1497 lrd_config_fname = get_beskeys(
"H5.LargeDataMemCacheFileName");
1500 if(lrd_config_fpath==
"" || lrd_config_fname==
"")
1507 string mcache_config_fname = lrd_config_fpath+
"/"+lrd_config_fname;
1511 ifstream mcache_config_file(mcache_config_fname.c_str());
1514 if(mcache_config_file.is_open()==
false){
1515 BESDEBUG(HDF5_NAME,
"The large data memory cache configure file "<<mcache_config_fname );
1516 BESDEBUG(HDF5_NAME,
" cannot be opened."<<endl);
1521 while(getline(mcache_config_file,temp_line)) {
1524 if(temp_line.size()>1 && temp_line.at(1)==
' ') {
1526 string subline = temp_line.substr(2);
1527 vector<string> temp_name_list;
1530 if(temp_line.at(0)==
'1') {
1531 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1533 lrd_cache_dir_list.insert(lrd_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1536 else if(temp_line.at(0)==
'0'){
1537 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1539 lrd_non_cache_dir_list.insert(lrd_non_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1542 else if(temp_line.at(0)==
'2') {
1548 for(
unsigned int i = 0; i<subline.size();i++){
1549 if(subline[i]==
'"') {
1550 dq_pos.push_back(i);
1552 else if(subline[i]==
'\'')
1553 sq_pos.push_back(i);
1555 if(dq_pos.size()==0 && sq_pos.size()==0)
1556 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1557 else if((dq_pos.size()!=0) &&(dq_pos.size()%2==0)&& sq_pos.size()==0) {
1558 unsigned int dq_index= 0;
1559 while(dq_index < dq_pos.size()){
1560 if(dq_pos[dq_index+1]>(dq_pos[dq_index]+1)) {
1561 temp_name_list.push_back
1562 (subline.substr(dq_pos[dq_index]+1,dq_pos[dq_index+1]-dq_pos[dq_index]-1));
1564 dq_index = dq_index + 2;
1567 else if((sq_pos.size()!=0) &&(sq_pos.size()%2==0)&& dq_pos.size()==0) {
1568 unsigned int sq_index= 0;
1569 while(sq_index < sq_pos.size()){
1570 if(sq_pos[sq_index+1]>(sq_pos[sq_index]+1)) {
1571 temp_name_list.push_back
1572 (subline.substr(sq_pos[sq_index]+1,sq_pos[sq_index+1]-sq_pos[sq_index]-1));
1574 sq_index = sq_index+2;
1578 lrd_var_cache_file_list.insert(lrd_var_cache_file_list.end(),temp_name_list.begin(),temp_name_list.end());
1586 for(
int i =0; i<lrd_cache_dir_list.size();i++)
1587 cerr<<
"lrd cache list is "<<lrd_cache_dir_list[i] <<endl;
1588 for(
int i =0; i<lrd_non_cache_dir_list.size();i++)
1589 cerr<<
"lrd non cache list is "<<lrd_non_cache_dir_list[i] <<endl;
1590 for(
int i =0; i<lrd_var_cache_file_list.size();i++)
1591 cerr<<
"lrd var cache file list is "<<lrd_var_cache_file_list[i] <<endl;
1595 mcache_config_file.close();
1596 if(lrd_cache_dir_list.size()==0 && lrd_non_cache_dir_list.size()==0 && lrd_var_cache_file_list.size()==0)
1603 bool HDF5RequestHandler::read_das_from_disk_cache(
const string & cache_filename,DAS *das_ptr) {
1605 BESDEBUG(HDF5_NAME,
"Coming to read_das_from_disk_cache() " << cache_filename << endl);
1606 bool ret_value =
true;
1607 FILE *md_file = NULL;
1608 md_file = fopen(cache_filename.c_str(),
"rb");
1610 if(NULL == md_file) {
1611 string bes_error =
"An error occurred trying to open a metadata cache file " + cache_filename;
1616 int fd_md = fileno(md_file);
1618 l_md = lock(F_RDLCK);
1621 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1624 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
1631 if(stat(cache_filename.c_str(),&sb) != 0) {
1632 string bes_error =
"An error occurred trying to stat a metadata cache file size " + cache_filename;
1638 size_t bytes_expected_read=(size_t)sb.st_size;
1639 BESDEBUG(HDF5_NAME,
"DAS Disk cache file size is " << bytes_expected_read << endl);
1642 buf.resize(bytes_expected_read);
1643 size_t bytes_to_read =fread((
void*)&buf[0],1,bytes_expected_read,md_file);
1644 if(bytes_to_read != bytes_expected_read)
1645 throw InternalErr(__FILE__,__LINE__,
"Fail to read the data from the das cache file.");
1647 char* temp_pointer =&buf[0];
1649 AttrTable*at = NULL;
1652 temp_pointer = get_attr_info_from_dc(temp_pointer,das_ptr,at);
1658 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1660 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1664 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a das cache file.");
1668 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1670 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1679 bool HDF5RequestHandler::write_dds_to_disk_cache(
const string& dds_cache_fname,DDS *dds_ptr) {
1681 BESDEBUG(HDF5_NAME,
"Write DDS to disk cache " << dds_cache_fname << endl);
1682 FILE *dds_file = fopen(dds_cache_fname.c_str(),
"w");
1684 if(NULL == dds_file) {
1685 string bes_error =
"An error occurred trying to open a metadata cache file " + dds_cache_fname;
1690 int fd_md = fileno(dds_file);
1692 l_md = lock(F_WRLCK);
1695 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1698 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
1703 dds_ptr->print(dds_file);
1706 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1708 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1712 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a dds cache file.");
1715 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1717 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1727 bool HDF5RequestHandler::write_das_to_disk_cache(
const string & das_cache_fname, DAS *das_ptr) {
1729 BESDEBUG(HDF5_NAME,
"Write DAS to disk cache " << das_cache_fname << endl);
1730 FILE *das_file = fopen(das_cache_fname.c_str(),
"wb");
1731 if(NULL == das_file) {
1732 string bes_error =
"An error occurred trying to open a metadata cache file " + das_cache_fname;
1736 int fd_md = fileno(das_file);
1738 l_md = lock(F_WRLCK);
1741 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1744 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
1749 write_das_to_file(das_ptr,das_file);
1752 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1754 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1758 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a dds cache file.");
1761 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1763 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1775 void write_das_to_file(DAS*das_ptr,FILE* das_file) {
1778 uint8_t category_flag = 2;
1779 AttrTable* top_table = das_ptr->get_top_level_attributes();
1780 write_das_table_to_file(top_table,das_file);
1783 fwrite((
const void*)&category_flag,1,1,das_file);
1789 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file) {
1791 if(temp_table !=NULL) {
1794 uint8_t category_flag = 2;
1797 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
1798 AttrTable::Attr_iter top_endit = temp_table->attr_end();
1799 AttrTable::Attr_iter top_it = top_startit;
1800 while(top_it !=top_endit) {
1801 AttrType atype = temp_table->get_attr_type(top_it);
1802 if(atype == Attr_unknown)
1803 throw InternalErr(__FILE__,__LINE__,
"Unsupported DAS Attribute type");
1804 else if(atype!=Attr_container) {
1805 BESDEBUG(HDF5_NAME,
"DAS to the disk cache, attr name is: "
1806 << temp_table->get_name(top_it) << endl);
1807 BESDEBUG(HDF5_NAME,
"DAS to the disk cache, attr type is: "
1808 << temp_table->get_type(top_it) << endl);
1817 write_das_attr_info(temp_table,temp_table->get_name(top_it),temp_table->get_type(top_it),das_file);
1820 BESDEBUG(HDF5_NAME,
"DAS to the disk cache, attr container name is: "
1821 << (*top_it)->name << endl);
1823 AttrTable* sub_table = temp_table->get_attr_table(top_it);
1824 write_container_name_to_file(sub_table->get_name(),das_file);
1825 write_das_table_to_file(sub_table,das_file);
1828 fwrite((
const void*)&category_flag,1,1,das_file);
1838 void write_container_name_to_file(
const string& cont_name,FILE *das_file) {
1841 uint8_t category_flag = 1;
1843 size_t bytes_to_write = cont_name.size()+
sizeof(size_t)+1;
1844 buf.resize(bytes_to_write);
1845 char*temp_pointer =&buf[0];
1846 memcpy((
void*)temp_pointer,(
void*)&category_flag,1);
1848 temp_pointer=copy_str(temp_pointer,cont_name);
1850 size_t bytes_to_be_written = fwrite((
const void*)&buf[0],1,bytes_to_write,das_file);
1851 if(bytes_to_be_written != bytes_to_write)
1852 throw InternalErr(__FILE__, __LINE__,
"Failed to write a DAS container name to a cache");
1858 void write_das_attr_info(AttrTable* dtp,
const string& attr_name,
const string & attr_type,FILE * das_file) {
1861 uint8_t category_flag = 0;
1863 unsigned int num_attr_elems = dtp->get_attr_num(attr_name);
1864 vector<string> attr_values;
1865 size_t total_attr_values_size = 0;
1866 for (
unsigned int i = 0; i <num_attr_elems;i++){
1867 attr_values.push_back((*(dtp->get_attr_vector(attr_name)))[i]);
1868 total_attr_values_size += attr_values[i].size();
1872 size_t bytes_to_write_attr = 1 + attr_name.size() + attr_type.size() + 2*
sizeof(size_t);
1878 bytes_to_write_attr +=
sizeof(
unsigned int) + num_attr_elems*
sizeof(
size_t)+total_attr_values_size;
1880 vector<char>attr_buf;
1881 attr_buf.resize(bytes_to_write_attr);
1882 char* temp_attrp =&attr_buf[0];
1885 memcpy((
void*)temp_attrp,(
void*)&category_flag,1);
1889 temp_attrp=copy_str(temp_attrp,attr_name);
1890 temp_attrp=copy_str(temp_attrp,attr_type);
1893 memcpy((
void*)temp_attrp,(
void*)&num_attr_elems,
sizeof(
unsigned int));
1894 temp_attrp+=
sizeof(
unsigned int);
1897 for (
unsigned int i = 0; i <num_attr_elems;i++)
1898 temp_attrp=copy_str(temp_attrp,(*(dtp->get_attr_vector(attr_name)))[i]);
1900 size_t bytes_to_be_written = fwrite((
const void*)&attr_buf[0],1,bytes_to_write_attr,das_file);
1901 if(bytes_to_be_written != bytes_to_write_attr)
1902 throw InternalErr(__FILE__, __LINE__,
"Failed to write a DAS attribute to a cache");
1910 bool build_data,
const string & container_name,
const string & h5_fname,
1911 const string & dds_cache_fname,
const string &das_cache_fname, hid_t h5_fd,
1915 BESDEBUG(HDF5_NAME,
"Read DDS from disk cache " << dds_cache_fname << endl);
1918 if(
true == build_data)
1919 dds = data_bdds->get_dds();
1925 DDS tdds(&tf,name_path(h5_fname),
"3.2");
1926 tdds.filename(h5_fname);
1928 FILE *dds_file = fopen(dds_cache_fname.c_str(),
"r");
1929 tdds.parse(dds_file);
1930 DDS* cache_dds =
new DDS(tdds);
1932 cerr<<
"before dds "<<endl;
1934 cerr<<
"after dds "<<endl;
1935 cerr<<
"before tdds "<<endl;
1936 cache_dds->dump(cerr);
1937 cerr<<
"after tdds "<<endl;
1942 Ancillary::read_ancillary_dds( *cache_dds, h5_fname ) ;
1944 add_das_to_dds(cache_dds,container_name,h5_fname,das_cache_fname,h5_fd,das_from_dc);
1945 if(
true == build_data)
1946 data_bdds->
set_dds(cache_dds);
1953 BESDEBUG(HDF5_NAME,
"Reading DDS from Disk Cache routine, For memory cache, DDS added to the cache for : " << h5_fname << endl);
1954 dds_cache->
add(
new DDS(*cache_dds), h5_fname);
1960 void HDF5RequestHandler::add_das_to_dds(DDS *dds,
const string &,
const string &filename,
1961 const string &das_cache_fname, hid_t h5_fd,
bool das_from_dc) {
1963 BESDEBUG(HDF5_NAME,
"Coming to add_das_to_dds() " << endl);
1967 bool use_das_cache =
false;
1969 das =
static_cast<DAS*
>(das_cache->
get(filename));
1971 use_das_cache =
true;
1973 if (
true == use_das_cache) {
1974 BESDEBUG(HDF5_NAME,
"DAS Cached hit for : " << filename << endl);
1975 dds->transfer_attributes(das);
1982 if (!container_name.empty())
1983 das->container_name(container_name);
1985 if(das_from_dc ==
true)
1986 read_das_from_disk_cache(das_cache_fname,das);
1989 bool h5_file_open =
true;
1991 h5_file_open =
false;
1992 if (
true == _usecf) {
1994 if(h5_file_open ==
false)
1995 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1997 read_cfdas( *das,filename,h5_fd);
1998 if(h5_file_open ==
false)
2002 if(h5_file_open ==
false)
2006 if(h5_file_open ==
false)
2010 Ancillary::read_ancillary_das( *das, filename ) ;
2012 if(das_cache_fname!=
"" && das_from_dc ==
false)
2013 write_das_to_disk_cache(das_cache_fname,das);
2016 dds->transfer_attributes(das);
2020 BESDEBUG(HDF5_NAME,
"Reading DDS from Disk Cache routine, For memory cache, DAS added to the cache for : " << filename << endl);
2021 das_cache->
add(
new DAS(*das), filename);
2029 bool check_beskeys(
const string key) {
2033 const string dosettrue =
"true";
2034 const string dosetyes =
"yes";
2037 if(
true == found ) {
2039 if( dosettrue == doset || dosetyes == doset )
2048 static unsigned int get_uint_key(
const string &key,
unsigned int def_val)
2054 if (
true == found) {
2056 return atoi(doset.c_str());
2063 static unsigned long get_ulong_key(
const string &key,
unsigned long def_val)
2069 if (
true == found) {
2071 return atol(doset.c_str());
2077 static float get_float_key(
const string &key,
float def_val)
2083 if (
true == found) {
2084 return atof(doset.c_str());
2091 static string get_beskeys(
const string &key) {
2094 string ret_value =
"";
2102 char* copy_str(
char*temp_ptr,
const string & str) {
2104 size_t str_size=str.size();
2105 memcpy((
void*)temp_ptr,(
void*)&str_size,
sizeof(
size_t));
2106 temp_ptr+=
sizeof(size_t);
2107 vector<char>temp_vc2(str.begin(),str.end());
2108 memcpy((
void*)temp_ptr,(
void*)&temp_vc2[0],str.size());
2109 temp_ptr+=str.size();
2118 char* obtain_str(
char*temp_ptr,
string & str) {
2120 size_t oname_size = *((
size_t *)temp_ptr);
2121 temp_ptr = temp_ptr +
sizeof(size_t);
2123 for(
unsigned int i =0; i<oname_size; i++){
2124 oname.push_back(*temp_ptr);
2135 char* get_attr_info_from_dc(
char*temp_pointer,DAS *das,AttrTable *at_par) {
2140 flag = *((uint8_t*)(temp_pointer));
2141 BESDEBUG(HDF5_NAME,
"Build DAS from the disk cache file flag: "
2142 <<
" flag = 0, attribute; flag = 1, container; flag =2; end of container;"
2143 <<
" flag = 3; the initial value to get the attribute retrieval process started."
2144 <<
" The flag value is "
2145 << (
int)flag <<endl);
2149 string container_name;
2150 temp_pointer = obtain_str(temp_pointer,container_name);
2151 BESDEBUG(HDF5_NAME,
"DAS from the disk cache, container name is " << container_name << endl);
2154 AttrTable*temp_at_par = at_par;
2156 at_par = das->add_table(container_name,
new AttrTable);
2158 at_par = at_par->append_container(container_name);
2160 temp_pointer = get_attr_info_from_dc(temp_pointer,das,at_par);
2162 at_par = temp_at_par;
2165 else if(flag == 0) {
2168 throw BESInternalError(
"The AttrTable must exist for DAS attributes", __FILE__, __LINE__ ) ;
2172 temp_pointer = obtain_str(temp_pointer,attr_name);
2173 BESDEBUG(HDF5_NAME,
"DAS from the disk cache, attr name is: " << attr_name << endl);
2177 temp_pointer = obtain_str(temp_pointer,attr_type);
2178 BESDEBUG(HDF5_NAME,
"DAS from the disk cache, attr type is: " << attr_type << endl);
2181 unsigned int num_values = *((
unsigned int*)(temp_pointer));
2182 BESDEBUG(HDF5_NAME,
"DAS from the disk cache, number of attribute values is: " << num_values << endl);
2183 temp_pointer+=
sizeof(
unsigned int);
2185 vector <string> attr_values;
2187 for(
unsigned int i = 0; i<num_values; i++) {
2189 temp_pointer = obtain_str(temp_pointer,attr_value);
2190 attr_values.push_back(attr_value);
2191 BESDEBUG(HDF5_NAME,
"DAS from the disk cache, attribute value is: " << attr_value << endl);
2194 at_par->append_attr(attr_name,attr_type,&attr_values);
2198 return temp_pointer;
2203 void get_attr_contents(AttrTable*temp_table) {
2204 if(temp_table !=NULL) {
2205 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2206 AttrTable::Attr_iter top_endit = temp_table->attr_end();
2207 AttrTable::Attr_iter top_it = top_startit;
2208 while(top_it !=top_endit) {
2209 AttrType atype = temp_table->get_attr_type(top_it);
2210 if(atype == Attr_unknown)
2211 cerr<<
"unsupported DAS attributes" <<endl;
2212 else if(atype!=Attr_container) {
2214 cerr<<
"Attribute name is "<<temp_table->get_name(top_it)<<endl;
2215 cerr<<
"Attribute type is "<<temp_table->get_type(top_it)<<endl;
2216 unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2217 cerr<<
"Attribute values are "<<endl;
2218 for (
unsigned int i = 0; i <num_attrs;i++)
2219 cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<
" ";
2223 cerr<<
"Coming to the attribute container. "<<endl;
2224 cerr<<
"container name is "<<(*top_it)->name <<endl;
2225 AttrTable* sub_table = temp_table->get_attr_table(top_it);
2226 cerr<<
"container table name is "<<sub_table->get_name() <<endl;
2227 get_attr_contents(sub_table);
2243 DDS *dds = bdds->get_dds();
2247 if (das_cache && (das =
static_cast<DAS*
>(das_cache->
get(filename)))) {
2248 BESDEBUG(HDF5_NAME,
"DAS Cached hit for : " << filename << endl);
2249 dds->transfer_attributes(das);
2255 if (!container_name.empty()) das->container_name(container_name);
2258 if (
true == _usecf) {
2260 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2262 read_cfdas( *das,filename,h5_fd);
2274 Ancillary::read_ancillary_das(*das, filename);
2276 dds->transfer_attributes(das);
2281 BESDEBUG(HDF5_NAME,
"DAS added to the cache for : " << filename << endl);
2282 das_cache->
add(das, filename);
2288 BESDEBUG(HDF5_NAME,
"Data ACCESS in add_attributes(): set the including attribute flag to true: "<<filename << endl);
2289 bdds->set_ia_flag(
true);
include the entry functions to execute the handlers
std::string get_symbolic_name() const
retrieve the symbolic name for this container
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Holds a DDS object within the BES.
virtual void set_container(const std::string &cn)
set the container in the DAP response object
void set_dds(libdap::DDS *ddsIn)
virtual void clear_container()
clear the container in the DAP response object
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
void set_dds(libdap::DDS *ddsIn)
virtual void set_container(const std::string &cn)
set the container in the DAP response object
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
BESContainer * container
pointer to current container in this interface
informational response object
virtual void add_data(const std::string &s)
add data to this informational object. If buffering is not set then the information is output directl...
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Represents a specific data type request handler.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
static std::string lowercase(const std::string &s)
static std::string implode(const std::list< std::string > &values, char delim)
An in-memory cache for DapObj (DAS, DDS, ...) objects.
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
static TheBESKeys * TheKeys()
Helper functions for generating DAS attributes and a function to check BES Key.
void depth_first(hid_t pid, const char *gname, DAS &das)
void find_gloattr(hid_t file, DAS &das)
bool breadth_first(hid_t pid, char *gname, D4Group *par_grp, const char *fname, bool use_dimscale)
hid_t get_fileid(const char *filename)
void close_fileid(hid_t fid)
The main header of the HDF5 OPeNDAP handler.