xrootd
XrdPfcInfo.hh
Go to the documentation of this file.
1 #ifndef __XRDPFC_INFO_HH__
2 #define __XRDPFC_INFO_HH__
3 //----------------------------------------------------------------------------------
4 // Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University
5 // Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman
6 //----------------------------------------------------------------------------------
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //----------------------------------------------------------------------------------
20 
21 #include <stdio.h>
22 #include <time.h>
23 #include <assert.h>
24 #include <vector>
25 
26 #include "XrdSys/XrdSysPthread.hh"
27 #include "XrdCl/XrdClConstants.hh"
28 #include "XrdCl/XrdClDefaultEnv.hh"
29 
30 class XrdOssDF;
31 class XrdCksCalc;
32 class XrdSysTrace;
33 
34 
35 namespace XrdCl
36 {
37 class Log;
38 }
39 
40 namespace XrdPfc
41 {
42 class Stats;
43 
44 //----------------------------------------------------------------------------
46 //----------------------------------------------------------------------------
47 
48 class Info
49 {
50 public:
52  struct AStat
53  {
54  time_t AttachTime;
55  time_t DetachTime;
56  int NumIos;
57  int Duration;
58  int NumMerged;
59  long long BytesHit;
60  long long BytesMissed;
61  long long BytesBypassed;
62 
63  AStat() :
64  AttachTime(0), DetachTime(0), NumIos(0), Duration(0), NumMerged(0),
66  {}
67 
68  void MergeWith(const AStat &a);
69  };
70 
71  struct Store
72  {
73  int m_version;
74  long long m_buffer_size;
75  long long m_file_size;
76  unsigned char *m_buff_synced;
77  char m_cksum[16];
78  time_t m_creationTime;
79  size_t m_accessCnt;
80  std::vector<AStat> m_astats;
81 
83  };
84 
85 
86  //------------------------------------------------------------------------
88  //------------------------------------------------------------------------
89  Info(XrdSysTrace* trace, bool prefetchBuffer = false);
90 
91  //------------------------------------------------------------------------
93  //------------------------------------------------------------------------
94  ~Info();
95 
96  //---------------------------------------------------------------------
98  //---------------------------------------------------------------------
99  void SetBitWritten(int i);
100 
101  //---------------------------------------------------------------------
103  //---------------------------------------------------------------------
104  bool TestBitWritten(int i) const;
105 
106  //---------------------------------------------------------------------
108  //---------------------------------------------------------------------
109  bool TestBitPrefetch(int i) const;
110 
111  //---------------------------------------------------------------------
113  //---------------------------------------------------------------------
114  void SetBitPrefetch(int i);
115 
116  //---------------------------------------------------------------------
118  //---------------------------------------------------------------------
119  void SetBitSynced(int i);
120 
121  //---------------------------------------------------------------------
123  //---------------------------------------------------------------------
125 
126  void SetBufferSize(long long);
127 
128  void SetFileSize(long long);
129 
130  //---------------------------------------------------------------------
134  //---------------------------------------------------------------------
135  void ResizeBits(int n);
136 
137  //---------------------------------------------------------------------
144  //---------------------------------------------------------------------
145  bool Read(XrdOssDF* fp, const std::string &fname = "<unknown>");
146 
147  //---------------------------------------------------------------------
150  //---------------------------------------------------------------------
151  bool Write(XrdOssDF* fp, const std::string &fname = "<unknown>");
152 
153  //---------------------------------------------------------------------
155  //---------------------------------------------------------------------
157 
158  //---------------------------------------------------------------------
160  //---------------------------------------------------------------------
162 
163  //---------------------------------------------------------------------
165  //---------------------------------------------------------------------
167 
168  //---------------------------------------------------------------------
170  //---------------------------------------------------------------------
172 
173  //---------------------------------------------------------------------
175  //---------------------------------------------------------------------
176  void WriteIOStat(Stats& s);
177 
178  //---------------------------------------------------------------------
180  //---------------------------------------------------------------------
182 
183  //---------------------------------------------------------------------
185  //---------------------------------------------------------------------
186  void WriteIOStatSingle(long long bytes_disk);
187 
188  //---------------------------------------------------------------------
190  //---------------------------------------------------------------------
191  void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc);
192 
193  //---------------------------------------------------------------------
195  //---------------------------------------------------------------------
196  bool IsAnythingEmptyInRng(int firstIdx, int lastIdx) const;
197 
198  //---------------------------------------------------------------------
200  //---------------------------------------------------------------------
201  int GetSizeInBytes() const;
202 
203  //---------------------------------------------------------------------
205  //---------------------------------------------------------------------
206  int GetSizeInBits() const;
207 
208  //---------------------------------------------------------------------
210  //---------------------------------------------------------------------
211  long long GetFileSize() const;
212 
213  //---------------------------------------------------------------------
215  //---------------------------------------------------------------------
216  bool GetLatestDetachTime(time_t& t) const;
217 
218  //---------------------------------------------------------------------
220  //---------------------------------------------------------------------
221  const AStat* GetLastAccessStats() const;
222 
223  //---------------------------------------------------------------------
225  //---------------------------------------------------------------------
226  long long GetBufferSize() const;
227 
228  //---------------------------------------------------------------------
230  //---------------------------------------------------------------------
231  bool IsComplete() const;
232 
233  //---------------------------------------------------------------------
235  //---------------------------------------------------------------------
236  int GetNDownloadedBlocks() const;
237 
238  //---------------------------------------------------------------------
240  //---------------------------------------------------------------------
241  long long GetNDownloadedBytes() const;
242 
243  //---------------------------------------------------------------------
245  //---------------------------------------------------------------------
246  int GetLastDownloadedBlock() const;
247 
248  //---------------------------------------------------------------------
250  //---------------------------------------------------------------------
251  long long GetExpectedDataFileSize() const;
252 
253  //---------------------------------------------------------------------
255  //---------------------------------------------------------------------
257 
258  //---------------------------------------------------------------------
260  //---------------------------------------------------------------------
261  size_t GetAccessCnt() const { return m_store.m_accessCnt; }
262 
263  //---------------------------------------------------------------------
265  //---------------------------------------------------------------------
266  int GetVersion() { return m_store.m_version; }
267 
268  //---------------------------------------------------------------------
270  //---------------------------------------------------------------------
271  const Store& RefStoredData() const { return m_store; }
272 
273  //---------------------------------------------------------------------
275  //---------------------------------------------------------------------
276  void GetCksum( unsigned char* buff, char* digest);
277 
278  static const char* m_traceID; // has to be m_ (convention in TRACE macros)
279  static const char* s_infoExtension;
280  static const int s_defaultVersion;
281  static size_t s_maxNumAccess; // can be set from configuration
282 
283  XrdSysTrace* GetTrace() const {return m_trace; }
284 
285 protected:
287 
290  unsigned char *m_buff_written;
291  unsigned char *m_buff_prefetch;
292 
294  bool m_complete;
295 
296 private:
297  inline unsigned char cfiBIT(int n) const { return 1 << n; }
298 
299  // Reading functions for older cinfo file formats
300  bool ReadV1(XrdOssDF* fp, const std::string &fname);
301  bool ReadV2(XrdOssDF* fp, const std::string &fname);
302 
304 };
305 
306 //------------------------------------------------------------------------------
307 
308 inline bool Info::TestBitWritten(int i) const
309 {
310  const int cn = i/8;
311  assert(cn < GetSizeInBytes());
312 
313  const int off = i - cn*8;
314  return (m_buff_written[cn] & cfiBIT(off)) != 0;
315 }
316 
317 inline void Info::SetBitWritten(int i)
318 {
319  const int cn = i/8;
320  assert(cn < GetSizeInBytes());
321 
322  const int off = i - cn*8;
323  m_buff_written[cn] |= cfiBIT(off);
324 }
325 
326 inline void Info::SetBitPrefetch(int i)
327 {
328  if (!m_buff_prefetch) return;
329 
330  const int cn = i/8;
331  assert(cn < GetSizeInBytes());
332 
333  const int off = i - cn*8;
334  m_buff_prefetch[cn] |= cfiBIT(off);
335 }
336 
337 inline bool Info::TestBitPrefetch(int i) const
338 {
339  if (!m_buff_prefetch) return false;
340 
341  const int cn = i/8;
342  assert(cn < GetSizeInBytes());
343 
344  const int off = i - cn*8;
345  return (m_buff_prefetch[cn] & cfiBIT(off)) != 0;
346 }
347 
348 inline void Info::SetBitSynced(int i)
349 {
350  const int cn = i/8;
351  assert(cn < GetSizeInBytes());
352 
353  const int off = i - cn*8;
354  m_store.m_buff_synced[cn] |= cfiBIT(off);
355 }
356 
357 //------------------------------------------------------------------------------
358 
359 inline int Info::GetNDownloadedBlocks() const
360 {
361  int cntd = 0;
362  for (int i = 0; i < m_sizeInBits; ++i)
363  if (TestBitWritten(i)) cntd++;
364 
365  return cntd;
366 }
367 
368 inline long long Info::GetNDownloadedBytes() const
369 {
371 }
372 
374 {
375  for (int i = m_sizeInBits - 1; i >= 0; --i)
376  if (TestBitWritten(i)) return i;
377 
378  return -1;
379 }
380 
381 inline long long Info::GetExpectedDataFileSize() const
382 {
383  int last_block = GetLastDownloadedBlock();
384  if (last_block == m_sizeInBits - 1)
385  return m_store.m_file_size;
386  else
387  return (last_block + 1) * m_store.m_buffer_size;
388 }
389 
390 inline int Info::GetSizeInBytes() const
391 {
392  if (m_sizeInBits)
393  return ((m_sizeInBits - 1)/8 + 1);
394  else
395  return 0;
396 }
397 
398 inline int Info::GetSizeInBits() const
399 {
400  return m_sizeInBits;
401 }
402 
403 inline long long Info::GetFileSize() const
404 {
405  return m_store.m_file_size;
406 }
407 
408 inline bool Info::IsComplete() const
409 {
410  return m_complete;
411 }
412 
413 inline bool Info::IsAnythingEmptyInRng(int firstIdx, int lastIdx) const
414 {
415  // TODO rewrite to use full byte comparisons outside of edges ?
416  // Also, it is always called with fisrtsdx = 0, lastIdx = m_sizeInBits.
417  for (int i = firstIdx; i < lastIdx; ++i)
418  if (! TestBitWritten(i)) return true;
419 
420  return false;
421 }
422 
424 {
426 }
427 
428 inline long long Info::GetBufferSize() const
429 {
430  return m_store.m_buffer_size;
431 }
432 
433 }
434 #endif
Definition: XrdCksCalc.hh:40
Definition: XrdOss.hh:62
Status of cached file. Can be read from and written into a binary file.
Definition: XrdPfcInfo.hh:49
void SetBitPrefetch(int i)
Mark block as obtained through prefetch.
Definition: XrdPfcInfo.hh:326
void UpdateDownloadCompleteStatus()
Update complete status.
Definition: XrdPfcInfo.hh:423
~Info()
Destructor.
void SetBitSynced(int i)
Mark block as synced to disk.
Definition: XrdPfcInfo.hh:348
Info(XrdSysTrace *trace, bool prefetchBuffer=false)
Constructor.
void SetAllBitsSynced()
Mark all blocks as synced to disk.
unsigned char * m_buff_prefetch
prefetch statistics
Definition: XrdPfcInfo.hh:291
void CompactifyAccessRecords()
Compactify access records to the configured maximum.
void WriteIOStatDetach(Stats &s)
Write close time together with bytes missed, hits, and disk.
static const int s_defaultVersion
Definition: XrdPfcInfo.hh:280
void ResizeBits(int n)
Reserve buffer for file_size / buffer_size bytes.
void WriteIOStat(Stats &s)
Write bytes missed, hits, and disk.
long long GetNDownloadedBytes() const
Get number of downloaded bytes.
Definition: XrdPfcInfo.hh:368
int GetLastDownloadedBlock() const
Get number of the last downloaded block.
Definition: XrdPfcInfo.hh:373
XrdCksCalc * m_cksCalc
Definition: XrdPfcInfo.hh:303
void SetBufferSize(long long)
unsigned char * m_buff_written
download state vector
Definition: XrdPfcInfo.hh:290
bool m_complete
cached
Definition: XrdPfcInfo.hh:294
bool TestBitPrefetch(int i) const
Test if block at the given index has been prefetched.
Definition: XrdPfcInfo.hh:337
bool IsComplete() const
Get complete status.
Definition: XrdPfcInfo.hh:408
bool Read(XrdOssDF *fp, const std::string &fname="<unknown>")
Rea load content from cinfo file into this object.
bool ReadV1(XrdOssDF *fp, const std::string &fname)
void ResetAllAccessStats()
Reset IO Stats.
unsigned char cfiBIT(int n) const
Definition: XrdPfcInfo.hh:297
const Store & RefStoredData() const
Get stored data.
Definition: XrdPfcInfo.hh:271
void SetBitWritten(int i)
Mark block as written to disk.
Definition: XrdPfcInfo.hh:317
int GetSizeInBits() const
Get number of blocks represented in download-state bit-vector.
Definition: XrdPfcInfo.hh:398
int m_sizeInBits
cached
Definition: XrdPfcInfo.hh:293
long long GetBufferSize() const
Get prefetch buffer size.
Definition: XrdPfcInfo.hh:428
bool ReadV2(XrdOssDF *fp, const std::string &fname)
int GetSizeInBytes() const
Get size of download-state bit-vector in bytes.
Definition: XrdPfcInfo.hh:390
long long GetExpectedDataFileSize() const
Get expected data file size.
Definition: XrdPfcInfo.hh:381
bool TestBitWritten(int i) const
Test if block at the given index is written to disk.
Definition: XrdPfcInfo.hh:308
long long GetFileSize() const
Get file size.
Definition: XrdPfcInfo.hh:403
void SetFileSize(long long)
XrdSysTrace * m_trace
Definition: XrdPfcInfo.hh:286
static const char * s_infoExtension
Definition: XrdPfcInfo.hh:279
Store m_store
Definition: XrdPfcInfo.hh:288
const AStat * GetLastAccessStats() const
Get latest access stats.
bool GetLatestDetachTime(time_t &t) const
Get latest detach time.
bool Write(XrdOssDF *fp, const std::string &fname="<unknown>")
XrdSysTrace * GetTrace() const
Definition: XrdPfcInfo.hh:283
void WriteIOStatSingle(long long bytes_disk)
Write single open/close time for given bytes read from disk.
static const char * m_traceID
Definition: XrdPfcInfo.hh:278
void GetCksum(unsigned char *buff, char *digest)
Get md5 cksum.
static size_t s_maxNumAccess
Definition: XrdPfcInfo.hh:281
int GetVersion()
Get version.
Definition: XrdPfcInfo.hh:266
int GetNDownloadedBlocks() const
Get number of downloaded blocks.
Definition: XrdPfcInfo.hh:359
size_t GetAccessCnt() const
Get number of accesses.
Definition: XrdPfcInfo.hh:261
bool IsAnythingEmptyInRng(int firstIdx, int lastIdx) const
Check download status in given block range.
Definition: XrdPfcInfo.hh:413
void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc)
Write open/close with given time and bytes read from disk.
void DisableDownloadStatus()
Disable allocating, writing, and reading of download status.
void WriteIOStatAttach()
Write open time in the last entry of access statistics.
bool m_hasPrefetchBuffer
constains current prefetch score
Definition: XrdPfcInfo.hh:289
Statistics of cache utilisation by a File object.
Definition: XrdPfcStats.hh:31
Definition: XrdSysTrace.hh:49
Definition: XrdClAnyObject.hh:26
Definition: XrdPfc.hh:46
Access statistics.
Definition: XrdPfcInfo.hh:53
long long BytesHit
read from cache
Definition: XrdPfcInfo.hh:59
void MergeWith(const AStat &a)
long long BytesBypassed
read from remote and dropped
Definition: XrdPfcInfo.hh:61
AStat()
Definition: XrdPfcInfo.hh:63
int Duration
total duration of all IOs attached
Definition: XrdPfcInfo.hh:57
int NumIos
number of IO objects attached during this access
Definition: XrdPfcInfo.hh:56
time_t DetachTime
close time
Definition: XrdPfcInfo.hh:55
long long BytesMissed
read from remote and cached
Definition: XrdPfcInfo.hh:60
time_t AttachTime
open time
Definition: XrdPfcInfo.hh:54
int NumMerged
number of times the record has been merged
Definition: XrdPfcInfo.hh:58
Definition: XrdPfcInfo.hh:72
Store()
Definition: XrdPfcInfo.hh:82
char m_cksum[16]
cksum of downloaded information
Definition: XrdPfcInfo.hh:77
int m_version
info version
Definition: XrdPfcInfo.hh:73
long long m_buffer_size
prefetch buffer size
Definition: XrdPfcInfo.hh:74
unsigned char * m_buff_synced
disk written state vector
Definition: XrdPfcInfo.hh:76
std::vector< AStat > m_astats
access records
Definition: XrdPfcInfo.hh:80
size_t m_accessCnt
total access count for the file
Definition: XrdPfcInfo.hh:79
long long m_file_size
number of file blocks
Definition: XrdPfcInfo.hh:75
time_t m_creationTime
time the info file was created
Definition: XrdPfcInfo.hh:78