XRootD
Loading...
Searching...
No Matches
XrdPfcInfo.hh
Go to the documentation of this file.
1#ifndef __XRDPFC_INFO_HH__
2#define __XRDPFC_INFO_HH__
3//----------------------------------------------------------------------------------
4// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University
5// Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman
6//----------------------------------------------------------------------------------
7// XRootD is free software: you can redistribute it and/or modify
8// it under the terms of the GNU Lesser General Public License as published by
9// the Free Software Foundation, either version 3 of the License, or
10// (at your option) any later version.
11//
12// XRootD is distributed in the hope that it will be useful,
13// but WITHOUT ANY WARRANTY; without even the implied warranty of
14// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15// GNU General Public License for more details.
16//
17// You should have received a copy of the GNU Lesser General Public License
18// along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19//----------------------------------------------------------------------------------
20
21#include <cstdio>
22#include <ctime>
23#include <assert.h>
24#include <vector>
25
29
30#include "XrdPfcTypes.hh"
31
32class XrdOssDF;
33class XrdCksCalc;
34class XrdSysTrace;
35
36namespace XrdPfc
37{
38class Stats;
39
40//----------------------------------------------------------------------------
42//----------------------------------------------------------------------------
43
44class Info
45{
46public:
47 struct Status {
48 union {
49 struct {
50 int f_cksum_check : 3;
51
52 int _free_bits_ : 29;
53 };
54 unsigned int _raw_;
55 };
56 Status() : _raw_(0) {}
57 };
58
60 struct AStat
61 {
62 time_t AttachTime;
63 time_t DetachTime;
64 int NumIos;
68 long long BytesHit;
69 long long BytesMissed;
70 long long BytesBypassed;
71
75 {}
76
77 void MergeWith(const AStat &a);
78 };
79
80 struct Store
81 {
82 long long m_buffer_size;
83 long long m_file_size;
86 size_t m_accessCnt;
89
90 Store () :
93 {}
94 };
95
96
97 //------------------------------------------------------------------------
99 //------------------------------------------------------------------------
100 Info(XrdSysTrace* trace, bool prefetchBuffer = false);
101
102 //------------------------------------------------------------------------
104 //------------------------------------------------------------------------
105 ~Info();
106
107 //---------------------------------------------------------------------
109 //---------------------------------------------------------------------
110 void SetBitWritten(int i);
111
112 //---------------------------------------------------------------------
114 //---------------------------------------------------------------------
115 bool TestBitWritten(int i) const;
116
117 //---------------------------------------------------------------------
119 //---------------------------------------------------------------------
120 bool TestBitPrefetch(int i) const;
121
122 //---------------------------------------------------------------------
124 //---------------------------------------------------------------------
125 void SetBitPrefetch(int i);
126
127 //---------------------------------------------------------------------
129 //---------------------------------------------------------------------
130 void SetBitSynced(int i);
131
132 //---------------------------------------------------------------------
134 //---------------------------------------------------------------------
135 void SetAllBitsSynced();
136
137 void SetBufferSizeFileSizeAndCreationTime(long long bs, long long fs);
138
139 //---------------------------------------------------------------------
141 //---------------------------------------------------------------------
142 void ResizeBits();
143
144 //---------------------------------------------------------------------
150 //---------------------------------------------------------------------
151 bool Read(XrdOssDF* fp, const char *dname, const char *fname = 0);
152
153 //---------------------------------------------------------------------
159 //---------------------------------------------------------------------
160 bool Write(XrdOssDF* fp, const char *dname, const char *fname = 0);
161
162 //---------------------------------------------------------------------
164 //---------------------------------------------------------------------
166
167 //---------------------------------------------------------------------
169 //---------------------------------------------------------------------
170 void ResetAllAccessStats();
171
172 //---------------------------------------------------------------------
174 //---------------------------------------------------------------------
175 void WriteIOStatAttach();
176
177 //---------------------------------------------------------------------
179 //---------------------------------------------------------------------
180 void WriteIOStat(Stats& s);
181
182 //---------------------------------------------------------------------
184 //---------------------------------------------------------------------
185 void WriteIOStatDetach(Stats& s);
186
187 //---------------------------------------------------------------------
189 //---------------------------------------------------------------------
190 void WriteIOStatSingle(long long bytes_disk);
191
192 //---------------------------------------------------------------------
194 //---------------------------------------------------------------------
195 void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc);
196
197 //---------------------------------------------------------------------
199 //---------------------------------------------------------------------
200 int CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const;
201
202 //---------------------------------------------------------------------
204 //---------------------------------------------------------------------
205 int GetBitvecSizeInBytes() const;
206
207 //---------------------------------------------------------------------
209 //---------------------------------------------------------------------
210 int GetNBlocks() const;
211
212 //---------------------------------------------------------------------
214 //---------------------------------------------------------------------
215 long long GetFileSize() const;
216
217 //---------------------------------------------------------------------
219 //---------------------------------------------------------------------
220 bool GetLatestDetachTime(time_t& t) const;
221
222 //---------------------------------------------------------------------
224 //---------------------------------------------------------------------
225 const AStat* GetLastAccessStats() const;
226
227 //---------------------------------------------------------------------
229 //---------------------------------------------------------------------
230 long long GetBufferSize() const;
231
232 //---------------------------------------------------------------------
234 //---------------------------------------------------------------------
235 bool IsComplete() const;
236
237 //---------------------------------------------------------------------
239 //---------------------------------------------------------------------
240 int GetNDownloadedBlocks() const;
241
242 //---------------------------------------------------------------------
244 //---------------------------------------------------------------------
245 long long GetNDownloadedBytes() const;
246
247 //---------------------------------------------------------------------
249 //---------------------------------------------------------------------
250 int GetLastDownloadedBlock() const;
251
252 //---------------------------------------------------------------------
254 //---------------------------------------------------------------------
255 long long GetExpectedDataFileSize() const;
256
257 //---------------------------------------------------------------------
259 //---------------------------------------------------------------------
261
262 //---------------------------------------------------------------------
264 //---------------------------------------------------------------------
265 size_t GetAccessCnt() const { return m_store.m_accessCnt; }
266
267 //---------------------------------------------------------------------
269 //---------------------------------------------------------------------
270 int GetVersion() { return m_version; }
271
272 //---------------------------------------------------------------------
274 //---------------------------------------------------------------------
275 const Store& RefStoredData() const { return m_store; }
276 const std::vector<AStat>& RefAStats() const { return m_astats; }
277
278 //---------------------------------------------------------------------
280 //---------------------------------------------------------------------
281 time_t GetCreationTime() const { return m_store.m_creationTime; }
282
283 //---------------------------------------------------------------------
285 //---------------------------------------------------------------------
286 uint32_t CalcCksumStore();
287 uint32_t CalcCksumSyncedAndAStats();
288 void CalcCksumMd5(unsigned char* buff, char* digest);
289
290 CkSumCheck_e GetCkSumState() const { return (CkSumCheck_e) m_store.m_status.f_cksum_check; }
291 const char* GetCkSumStateAsText() const;
292
293 bool IsCkSumCache() const { return m_store.m_status.f_cksum_check & CSChk_Cache; }
294 bool IsCkSumNet() const { return m_store.m_status.f_cksum_check & CSChk_Net; }
295 bool IsCkSumAny() const { return m_store.m_status.f_cksum_check & CSChk_Both; }
296 bool IsCkSumBoth() const { return (m_store.m_status.f_cksum_check & CSChk_Both) == CSChk_Both; }
297
298 void SetCkSumState(CkSumCheck_e css) { m_store.m_status.f_cksum_check = css; }
299 void DowngradeCkSumState(CkSumCheck_e css_ref) { m_store.m_status.f_cksum_check &= css_ref; }
300 void ResetCkSumCache();
301 void ResetCkSumNet();
302
303 bool HasNoCkSumTime() const { return m_store.m_noCkSumTime != 0; }
304 time_t GetNoCkSumTime() const { return m_store.m_noCkSumTime; }
307
308#ifdef XRDPFC_CKSUM_TEST
309 static void TestCksumStuff();
310#endif
311
312 static const char* m_traceID; // has to be m_ (convention in TRACE macros)
313 static const char* s_infoExtension;
314 static const size_t s_infoExtensionLen;
315 static size_t s_maxNumAccess; // can be set from configuration
316 static const int s_defaultVersion;
317
318 XrdSysTrace* GetTrace() const {return m_trace; }
319
320protected:
322
324 unsigned char *m_buff_synced;
325 unsigned char *m_buff_written;
326 unsigned char *m_buff_prefetch;
327 std::vector<AStat> m_astats;
328
334
335private:
336 inline unsigned char cfiBIT(int n) const { return 1 << n; }
337
338 // Reading functions for older cinfo file formats
339 bool ReadV2(XrdOssDF* fp, off_t off, const char *dname, const char *fname);
340 bool ReadV3(XrdOssDF* fp, off_t off, const char *dname, const char *fname);
341
342 XrdCksCalc* m_cksCalcMd5;
343};
344
345//------------------------------------------------------------------------------
346
347inline bool Info::TestBitWritten(int i) const
348{
349 const int cn = i/8;
350 assert(cn < GetBitvecSizeInBytes());
351
352 const int off = i - cn*8;
353 return (m_buff_written[cn] & cfiBIT(off)) != 0;
354}
355
356inline void Info::SetBitWritten(int i)
357{
358 const int cn = i/8;
359 assert(cn < GetBitvecSizeInBytes());
360
361 const int off = i - cn*8;
362
363 m_buff_written[cn] |= cfiBIT(off);
364
365 if (--m_missingBlocks == 0)
366 m_complete = true;
367}
368
369inline void Info::SetBitPrefetch(int i)
370{
371 if (!m_buff_prefetch) return;
372
373 const int cn = i/8;
374 assert(cn < GetBitvecSizeInBytes());
375
376 const int off = i - cn*8;
377 m_buff_prefetch[cn] |= cfiBIT(off);
378}
379
380inline bool Info::TestBitPrefetch(int i) const
381{
382 if (!m_buff_prefetch) return false;
383
384 const int cn = i/8;
385 assert(cn < GetBitvecSizeInBytes());
386
387 const int off = i - cn*8;
388 return (m_buff_prefetch[cn] & cfiBIT(off)) != 0;
389}
390
391inline void Info::SetBitSynced(int i)
392{
393 const int cn = i/8;
394 assert(cn < GetBitvecSizeInBytes());
395
396 const int off = i - cn*8;
397 m_buff_synced[cn] |= cfiBIT(off);
398}
399
400//------------------------------------------------------------------------------
401
403{
404 int cntd = 0;
405 for (int i = 0; i < m_bitvecSizeInBits; ++i)
406 if (TestBitWritten(i)) cntd++;
407
408 return cntd;
409}
410
411inline long long Info::GetNDownloadedBytes() const
412{
414}
415
417{
418 for (int i = m_bitvecSizeInBits - 1; i >= 0; --i)
419 if (TestBitWritten(i)) return i;
420
421 return -1;
422}
423
424inline long long Info::GetExpectedDataFileSize() const
425{
426 int last_block = GetLastDownloadedBlock();
427 if (last_block == m_bitvecSizeInBits - 1)
428 return m_store.m_file_size;
429 else
430 return (last_block + 1) * m_store.m_buffer_size;
431}
432
434{
436 return ((m_bitvecSizeInBits - 1)/8 + 1);
437 else
438 return 0;
439}
440
441inline int Info::GetNBlocks() const
442{
443 return m_bitvecSizeInBits;
444}
445
446inline long long Info::GetFileSize() const
447{
448 return m_store.m_file_size;
449}
450
451inline bool Info::IsComplete() const
452{
453 return m_complete;
454}
455
456inline int Info::CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const
457{
458 // TODO rewrite to use full byte comparisons outside of edges ?
459 // Also, it seems to be always called with firstIdx = 0, lastIdx = m_bitvecSizeInBits.
460 int cnt = 0;
461 for (int i = firstIdx; i < lastIdx; ++i)
462 if (! TestBitWritten(i)) ++cnt;
463
464 return cnt;
465}
466
468{
471}
472
473inline long long Info::GetBufferSize() const
474{
475 return m_store.m_buffer_size;
476}
477
478}
479#endif
Status of cached file. Can be read from and written into a binary file.
Definition XrdPfcInfo.hh:45
void SetBitPrefetch(int i)
Mark block as obtained through prefetch.
void UpdateDownloadCompleteStatus()
Update complete status.
bool IsCkSumAny() const
static const char * s_infoExtension
uint32_t CalcCksumStore()
Get cksum, MD5 is for backward compatibility with V2 and V3.
void ResizeBits()
Reserve bit vectors for file_size / buffer_size bytes.
void ResetCkSumCache()
static const int s_defaultVersion
const std::vector< AStat > & RefAStats() const
void SetBitSynced(int i)
Mark block as synced to disk.
void WriteIOStatSingle(long long bytes_disk)
Write single open/close time for given bytes read from disk.
int GetBitvecSizeInBytes() const
Get size of download-state bit-vector in bytes.
time_t GetNoCkSumTimeForUVKeep() const
unsigned char * m_buff_prefetch
prefetch statistics
static const size_t s_infoExtensionLen
CkSumCheck_e GetCkSumState() const
const AStat * GetLastAccessStats() const
Get latest access stats.
void WriteIOStatAttach()
Write open time in the last entry of access statistics.
bool GetLatestDetachTime(time_t &t) const
Get latest detach time.
void ResetCkSumNet()
bool Write(XrdOssDF *fp, const char *dname, const char *fname=0)
void DowngradeCkSumState(CkSumCheck_e css_ref)
long long GetNDownloadedBytes() const
Get number of downloaded bytes.
void CompactifyAccessRecords()
Compactify access records to the configured maximum.
int GetLastDownloadedBlock() const
Get number of the last downloaded block.
XrdSysTrace * GetTrace() const
const Store & RefStoredData() const
Get stored data.
uint32_t CalcCksumSyncedAndAStats()
~Info()
Destructor.
time_t GetCreationTime() const
Get file size.
unsigned char * m_buff_written
download state vector
bool IsCkSumNet() const
unsigned char * m_buff_synced
disk written state vector
bool m_complete
cached; if false, set to true when missingBlocks hit zero
void ResetAllAccessStats()
Reset IO Stats.
bool TestBitPrefetch(int i) const
Test if block at the given index has been prefetched.
bool IsComplete() const
Get complete status.
bool IsCkSumCache() const
void SetBitWritten(int i)
Mark block as written to disk.
time_t GetNoCkSumTime() const
int m_missingBlocks
cached, updated in SetBitWritten()
long long GetBufferSize() const
Get prefetch buffer size.
void WriteIOStat(Stats &s)
Write bytes missed, hits, and disk.
long long GetExpectedDataFileSize() const
Get expected data file size.
bool TestBitWritten(int i) const
Test if block at the given index is written to disk.
int m_bitvecSizeInBits
cached
long long GetFileSize() const
Get file size.
XrdSysTrace * m_trace
std::vector< AStat > m_astats
access records
void SetAllBitsSynced()
Mark all blocks as synced to disk.
bool Read(XrdOssDF *fp, const char *dname, const char *fname=0)
Read content of cinfo file into this object.
void SetCkSumState(CkSumCheck_e css)
const char * GetCkSumStateAsText() const
void ResetNoCkSumTime()
bool IsCkSumBoth() const
int GetVersion()
Get version.
int GetNDownloadedBlocks() const
Get number of downloaded blocks.
size_t GetAccessCnt() const
Get number of accesses.
int CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const
Check download status in given block range.
void SetBufferSizeFileSizeAndCreationTime(long long bs, long long fs)
void WriteIOStatDetach(Stats &s)
Write close time together with bytes missed, hits, and disk.
static size_t s_maxNumAccess
static const char * m_traceID
void CalcCksumMd5(unsigned char *buff, char *digest)
int GetNBlocks() const
Get number of blocks represented in download-state bit-vector.
bool m_hasPrefetchBuffer
constains current prefetch score
bool HasNoCkSumTime() const
Statistics of cache utilisation by a File object.
@ CSChk_Cache
XrdPosixStats Stats
Access statistics.
Definition XrdPfcInfo.hh:61
long long BytesHit
read from cache
Definition XrdPfcInfo.hh:68
long long BytesBypassed
read from remote and dropped
Definition XrdPfcInfo.hh:70
void MergeWith(const AStat &a)
int Duration
total duration of all IOs attached
Definition XrdPfcInfo.hh:65
int Reserved
reserved / alignment
Definition XrdPfcInfo.hh:67
int NumIos
number of IO objects attached during this access
Definition XrdPfcInfo.hh:64
time_t DetachTime
close time
Definition XrdPfcInfo.hh:63
long long BytesMissed
read from remote and cached
Definition XrdPfcInfo.hh:69
time_t AttachTime
open time
Definition XrdPfcInfo.hh:62
int NumMerged
number of times the record has been merged
Definition XrdPfcInfo.hh:66
int m_astatSize
size of AStat vector
Definition XrdPfcInfo.hh:88
Status m_status
status information
Definition XrdPfcInfo.hh:87
long long m_buffer_size
buffer / block size
Definition XrdPfcInfo.hh:82
time_t m_noCkSumTime
time when first non-cksummed block was detected
Definition XrdPfcInfo.hh:85
size_t m_accessCnt
total access count for the file
Definition XrdPfcInfo.hh:86
long long m_file_size
size of file in bytes
Definition XrdPfcInfo.hh:83
time_t m_creationTime
time the info file was created
Definition XrdPfcInfo.hh:84