casacore
ISMBase.h
Go to the documentation of this file.
1//# ISMBase.h: Base class of the Incremental Storage Manager
2//# Copyright (C) 1996,1997,1999,2000,2001,2002
3//# Associated Universities, Inc. Washington DC, USA.
4//#
5//# This library is free software; you can redistribute it and/or modify it
6//# under the terms of the GNU Library General Public License as published by
7//# the Free Software Foundation; either version 2 of the License, or (at your
8//# option) any later version.
9//#
10//# This library is distributed in the hope that it will be useful, but WITHOUT
11//# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12//# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13//# License for more details.
14//#
15//# You should have received a copy of the GNU Library General Public License
16//# along with this library; if not, write to the Free Software Foundation,
17//# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18//#
19//# Correspondence concerning AIPS++ should be addressed as follows:
20//# Internet email: aips2-request@nrao.edu.
21//# Postal address: AIPS++ Project Office
22//# National Radio Astronomy Observatory
23//# 520 Edgemont Road
24//# Charlottesville, VA 22903-2475 USA
25//#
26//# $Id$
27
28#ifndef TABLES_ISMBASE_H
29#define TABLES_ISMBASE_H
30
31
32//# Includes
33#include <casacore/casa/aips.h>
34#include <casacore/tables/DataMan/DataManager.h>
35#include <casacore/casa/Containers/Block.h>
36#include <casacore/casa/iosfwd.h>
37
38namespace casacore { //# NAMESPACE CASACORE - BEGIN
39
40//# Forward declarations
41class BucketCache;
42class BucketFile;
43class ISMBucket;
44class ISMIndex;
45class ISMColumn;
46class StManArrayFile;
47
48// <summary>
49// Base class of the Incremental Storage Manager
50// </summary>
51
52// <use visibility=local>
53
54// <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="tIncrementalStMan.cc">
55// </reviewed>
56
57// <prerequisite>
58//# Classes you should understand before using this one.
59// <li> <linkto class=IncrementalStMan>IncrementalStMan</linkto>
60// <li> <linkto class=ISMColumn>ISMColumn</linkto>
61// </prerequisite>
62
63// <etymology>
64// ISMBase is the base class of the Incremental Storage Manager.
65// </etymology>
66
67// <synopsis>
68// The behaviour of this class is described in
69// <linkto class="IncrementalStMan:description">IncrementalStMan</linkto>.
70
71// <motivation>
72// The public interface of ISMBase is quite large, because the other
73// internal ISM classes need these functions. To have a class with a
74// minimal interface for the normal user, class <src>IncrementalStMan</src>
75// is derived from it.
76// <br>IncrementalStMan needs an isA- instead of hasA-relation to be
77// able to bind columns to it in class <linkto class=SetupNewTable>
78// SetupNewTable</linkto>.
79// </motivation>
80
81// <todo asof="$DATE:$">
82//# A List of bugs, limitations, extensions or planned refinements.
83// <li> Removed AipsIO argument from open and close.
84// </todo>
85
86
87class ISMBase: public DataManager
88{
89public:
90 // Create an incremental storage manager without a name.
91 // The bucket size has to be given in bytes and the cache size in buckets.
92 // The bucket size is checked or calculated (if 0) as described in
93 // IncrementalStMan.h.
94 explicit ISMBase (uInt bucketSize = 0, Bool checkBucketSize = True,
95 uInt cacheSize = 1);
96
97 // Create an incremental storage manager with the given name.
98 // The bucket size has to be given in bytes and the cache size in buckets.
99 // The bucket size is checked or calculated (if 0) as described in
100 // IncrementalStMan.h.
102 uInt bucketSize, Bool checkBucketSize, uInt cacheSize);
103
104 // Create an incremental storage manager with the given name.
105 // The specifications are in the record (as created by dataManagerSpec).
106 ISMBase (const String& aDataManName,
107 const Record& spec);
108
110
111 // Clone this object.
112 // It does not clone ISMColumn objects possibly used.
113 // The caller has to delete the newly created object.
114 virtual DataManager* clone() const;
115
116 // Get the type name of the data manager (i.e. IncrementalStMan).
117 virtual String dataManagerType() const;
118
119 // Get the name given to the storage manager (in the constructor).
120 virtual String dataManagerName() const;
121
122 // Record a record containing data manager specifications.
123 virtual Record dataManagerSpec() const;
124
125 // Get data manager properties that can be modified.
126 // It is only ActualCacheSize (the actual cache size in buckets).
127 // It is a subset of the data manager specification.
128 virtual Record getProperties() const;
129
130 // Modify data manager properties.
131 // Only MaxCacheSize can be used. It is similar to function setCacheSize
132 // with <src>canExceedNrBuckets=False</src>.
133 virtual void setProperties (const Record& spec);
134
135 // Get the version of the class.
136 uInt version() const;
137
138 // Set the cache size (in buckets).
139 // If <src>canExceedNrBuckets=True</src>, the given cache size can be
140 // larger than the nr of buckets in the file. In this way the cache can
141 // be made large enough for a future file extnsion.
142 // Otherwise, it is limited to the actual number of buckets. This is useful
143 // if one wants the entire file to be cached.
144 void setCacheSize (uInt cacheSize, Bool canExceedNrBuckets);
145
146 // Get the current cache size (in buckets).
147 uInt cacheSize() const;
148
149 // Clear the cache used by this storage manager.
150 // It will flush the cache as needed and remove all buckets from it.
152
153 // Show the statistics of all caches used.
154 virtual void showCacheStatistics (ostream& os) const;
155
156 // Show the index statistics.
157 void showIndexStatistics (ostream& os);
158
159 // Show the layout of the buckets
160 void showBucketLayout (ostream& os);
161
162 // Get the bucket size (in bytes).
163 uInt bucketSize() const;
164
165 // Get the size of a uInt in external format (can be canonical or local).
166 uInt uIntSize() const;
167
168 // Get the size of a rownr in external format (can be canonical or local).
169 uInt rownrSize() const;
170
171 // Get the bucket containing the given row.
172 // Also return the first and last row of that bucket.
173 // The bucket object is created and deleted by the caching mechanism.
174 ISMBucket* getBucket (rownr_t rownr, rownr_t& bucketStartRow,
175 rownr_t& bucketNrrow);
176
177 // Get the next bucket.
178 // cursor=0 indicates the start of the iteration.
179 // The first bucket returned is the bucket containing the rownr
180 // given in bucketStartRow.
181 // After each iteration BucketStartRow and bucketNrrow are set.
182 // A 0 is returned when no more buckets.
183 // The bucket object is created and deleted by the caching mechanism.
184 ISMBucket* nextBucket (uInt& cursor, rownr_t& bucketStartRow,
185 rownr_t& bucketNrrow);
186
187 // Get access to the temporary buffer.
188 char* tempBuffer() const;
189
190 // Get a unique column number for the column
191 // (it is only unique for this storage manager).
192 // This is used by ISMColumnIndArr to create a unique file name.
193 uInt uniqueNr();
194
195 // Get the number of rows in this storage manager.
196 rownr_t nrow() const;
197
198 // Can the storage manager add rows? (yes)
199 virtual Bool canAddRow() const;
200
201 // Can the storage manager delete rows? (yes)
202 virtual Bool canRemoveRow() const;
203
204 // Can the storage manager add columns? (not yet)
205 virtual Bool canAddColumn() const;
206
207 // Can the storage manager delete columns? (not yet)
208 virtual Bool canRemoveColumn() const;
209
210 // Make the object from the type name string.
211 // This function gets registered in the DataManager "constructor" map.
212 // The caller has to delete the object.
214 const Record& spec);
215
216 // Get access to the given column.
217 ISMColumn& getColumn (uInt colnr);
218
219 // Add a bucket to the storage manager (i.e. to the cache).
220 // The pointer is taken over.
221 void addBucket (rownr_t rownr, ISMBucket* bucket);
222
223 // Make the current bucket in the cache dirty (i.e. something has been
224 // changed in it and it needs to be written when removed from the cache).
225 // (used by ISMColumn::putValue).
227
228 // Open (if needed) the file for indirect arrays with the given mode.
229 // Return a pointer to the object.
231
232 // Check that there are no repeated rowIds in the buckets comprising this ISM.
233 Bool checkBucketLayout (uInt& offendingCursor,
234 rownr_t& offendingBucketStartRow,
235 uInt& offendingBucketNrow,
236 uInt& offendingBucketNr,
237 uInt& offendingCol,
238 uInt& ffendingIndex,
239 rownr_t& offendingRow,
240 rownr_t& offendingPrevRow);
241
242private:
243 // Copy constructor (only meant for clone function).
244 ISMBase (const ISMBase& that);
245
246 // Assignment cannot be used.
248
249 // (Re)create the index, file, and cache object.
250 void recreate();
251
252 // The data manager supports use of MultiFile.
253 virtual Bool hasMultiFileSupport() const;
254
255 // Flush and optionally fsync the data.
256 // It returns a True status if it had to flush (i.e. if data have changed).
257 virtual Bool flush (AipsIO&, Bool fsync);
258
259 // Let the storage manager create files as needed for a new table.
260 // This allows a column with an indirect array to create its file.
261 virtual void create64 (rownr_t nrrow);
262
263 // Open the storage manager file for an existing table, read in
264 // the data, and let the ISMColumn objects read their data.
265 virtual rownr_t open64 (rownr_t nrrow, AipsIO&);
266
267 // Resync the storage manager with the new file contents.
268 // This is done by clearing the cache.
269 virtual rownr_t resync64 (rownr_t nrrow);
270
271 // Reopen the storage manager files for read/write.
272 virtual void reopenRW();
273
274 // The data manager will be deleted (because all its columns are
275 // requested to be deleted).
276 // So clean up the things needed (e.g. delete files).
277 virtual void deleteManager();
278
279 // Let the storage manager initialize itself.
280 // It is used by create and open.
281 void init();
282
283 // Add rows to the storage manager.
284 // Per column it extends the interval for which the last value written
285 // is valid.
286 virtual void addRow64 (rownr_t nrrow);
287
288 // Delete a row from all columns.
289 virtual void removeRow64 (rownr_t rownr);
290
291 // Do the final addition of a column.
292 // The <src>DataManagerColumn</src> object has already been created
293 // (by the <src>makeXXColumn</src> function) and added to
294 // <src>colSet_p</src>. However, it still has to be added to the
295 // data files, which is done by this function. It uses the
296 // pointer to find the correct column in the <src>colSet_p</src>.
298
299 // Remove a column from the data file and the <src>colSet_p</src>.
300 // The <src>DataManagerColumn</src> object gets deleted..
302
303 // Create a column in the storage manager on behalf of a table column.
304 // The caller has to delete the newly created object.
305 // <group>
306 // Create a scalar column.
308 int dataType,
309 const String& dataTypeID);
310 // Create a direct array column.
312 int dataType,
313 const String& dataTypeID);
314 // Create an indirect array column.
316 int dataType,
317 const String& dataTypeID);
318 // </group>
319
320 // Get the cache object.
321 // This will construct the cache object if not present yet.
322 // The cache object will be deleted by the destructor.
324
325 // Get the index object.
326 // This will construct the index object if not present yet.
327 // The index object will be deleted by the destructor.
329
330 // Construct the cache object (if not constructed yet).
331 void makeCache();
332
333 // Construct the index object (if not constructed yet) and read it.
334 void makeIndex();
335
336 // Read the index (at the end of the file).
337 void readIndex();
338
339 // Write the index (at the end of the file).
341
342
343 //# Declare member variables.
344 // Name of data manager.
346 // The version of the class.
348 // The file containing the indirect arrays.
350 // Unique nr for column in this storage manager.
352 // The number of rows in the columns.
354 // The assembly of all columns.
356 // The cache with the ISM buckets.
358 // The file containing all data.
360 // The ISM bucket index.
362 // The persistent cache size.
364 // The actual cache size.
366 // The initial number of buckets in the cache.
368 // The nr of free buckets.
370 // The first free bucket.
372 // The bucket size.
374 // Check a positive bucketsize?
376 // Has the data changed since the last flush?
378 // The size of a uInt in external format (local or canonical).
380 // The size of a rownr in external format (local or canonical).
382 // A temporary read/write buffer (also for other classes).
384};
385
386
387inline uInt ISMBase::version() const
388{
389 return version_p;
390}
391
393{
394 return cacheSize_p;
395}
396
398{
399 return uniqnr_p++;
400}
401
402inline rownr_t ISMBase::nrow() const
403{
404 return nrrow_p;
405}
406
408{
409 return bucketSize_p;
410}
411
413{
414 return uIntSize_p;
415}
416
418{
419 return rownrSize_p;
420}
421
422inline char* ISMBase::tempBuffer() const
423{
424 return tempBuffer_p;
425}
426
428{
429 if (cache_p == 0) {
430 makeCache();
431 }
432 return *cache_p;
433}
434
436{
437 if (index_p == 0) {
438 makeIndex();
439 }
440 return *index_p;
441}
442
444{
445 return *(colSet_p[colnr]);
446}
447
448
449
450} //# NAMESPACE CASACORE - END
451
452#endif
Cache for buckets in a part of a file.
Definition: BucketCache.h:218
OpenOption
Define the possible ByteIO open options.
Definition: ByteIO.h:65
Abstract base class for a data manager.
Definition: DataManager.h:221
rownr_t nrrow_p
The number of rows in the columns.
Definition: ISMBase.h:353
virtual Record getProperties() const
Get data manager properties that can be modified.
virtual void create64(rownr_t nrrow)
Let the storage manager create files as needed for a new table.
PtrBlock< ISMColumn * > colSet_p
The assembly of all columns.
Definition: ISMBase.h:355
void readIndex()
Read the index (at the end of the file).
virtual void setProperties(const Record &spec)
Modify data manager properties.
ISMBase(const ISMBase &that)
Copy constructor (only meant for clone function).
virtual rownr_t open64(rownr_t nrrow, AipsIO &)
Open the storage manager file for an existing table, read in the data, and let the ISMColumn objects ...
virtual void addColumn(DataManagerColumn *)
Do the final addition of a column.
StManArrayFile * openArrayFile(ByteIO::OpenOption opt)
Open (if needed) the file for indirect arrays with the given mode.
char * tempBuffer() const
Get access to the temporary buffer.
Definition: ISMBase.h:422
char * tempBuffer_p
A temporary read/write buffer (also for other classes).
Definition: ISMBase.h:383
uInt uniqnr_p
Unique nr for column in this storage manager.
Definition: ISMBase.h:351
ISMIndex * index_p
The ISM bucket index.
Definition: ISMBase.h:361
virtual Bool flush(AipsIO &, Bool fsync)
Flush and optionally fsync the data.
Bool checkBucketLayout(uInt &offendingCursor, rownr_t &offendingBucketStartRow, uInt &offendingBucketNrow, uInt &offendingBucketNr, uInt &offendingCol, uInt &ffendingIndex, rownr_t &offendingRow, rownr_t &offendingPrevRow)
Check that there are no repeated rowIds in the buckets comprising this ISM.
uInt bucketSize_p
The bucket size.
Definition: ISMBase.h:373
virtual Bool canAddRow() const
Can the storage manager add rows? (yes)
uInt bucketSize() const
Get the bucket size (in bytes).
Definition: ISMBase.h:407
rownr_t nrow() const
Get the number of rows in this storage manager.
Definition: ISMBase.h:402
virtual DataManager * clone() const
Clone this object.
virtual String dataManagerName() const
Get the name given to the storage manager (in the constructor).
ISMColumn & getColumn(uInt colnr)
Get access to the given column.
Definition: ISMBase.h:443
ISMIndex & getIndex()
Get the index object.
Definition: ISMBase.h:435
void setCacheSize(uInt cacheSize, Bool canExceedNrBuckets)
Set the cache size (in buckets).
void showIndexStatistics(ostream &os)
Show the index statistics.
void setBucketDirty()
Make the current bucket in the cache dirty (i.e.
uInt rownrSize() const
Get the size of a rownr in external format (can be canonical or local).
Definition: ISMBase.h:417
virtual String dataManagerType() const
Get the type name of the data manager (i.e.
uInt nbucketInit_p
The initial number of buckets in the cache.
Definition: ISMBase.h:367
uInt persCacheSize_p
The persistent cache size.
Definition: ISMBase.h:363
virtual Record dataManagerSpec() const
Record a record containing data manager specifications.
uInt version_p
The version of the class.
Definition: ISMBase.h:347
virtual void reopenRW()
Reopen the storage manager files for read/write.
void recreate()
(Re)create the index, file, and cache object.
Bool checkBucketSize_p
Check a positive bucketsize?
Definition: ISMBase.h:375
virtual void removeColumn(DataManagerColumn *)
Remove a column from the data file and the colSet_p.
StManArrayFile * iosfile_p
The file containing the indirect arrays.
Definition: ISMBase.h:349
void init()
Let the storage manager initialize itself.
BucketCache & getCache()
Get the cache object.
Definition: ISMBase.h:427
static DataManager * makeObject(const String &dataManagerType, const Record &spec)
Make the object from the type name string.
virtual void showCacheStatistics(ostream &os) const
Show the statistics of all caches used.
uInt uniqueNr()
Get a unique column number for the column (it is only unique for this storage manager).
Definition: ISMBase.h:397
void makeIndex()
Construct the index object (if not constructed yet) and read it.
virtual DataManagerColumn * makeIndArrColumn(const String &name, int dataType, const String &dataTypeID)
Create an indirect array column.
uInt nFreeBucket_p
The nr of free buckets.
Definition: ISMBase.h:369
uInt uIntSize_p
The size of a uInt in external format (local or canonical).
Definition: ISMBase.h:379
Bool dataChanged_p
Has the data changed since the last flush?
Definition: ISMBase.h:377
String dataManName_p
Name of data manager.
Definition: ISMBase.h:345
virtual rownr_t resync64(rownr_t nrrow)
Resync the storage manager with the new file contents.
virtual Bool canRemoveColumn() const
Can the storage manager delete columns? (not yet)
virtual DataManagerColumn * makeDirArrColumn(const String &name, int dataType, const String &dataTypeID)
Create a direct array column.
virtual DataManagerColumn * makeScalarColumn(const String &name, int dataType, const String &dataTypeID)
Create a column in the storage manager on behalf of a table column.
ISMBase(const String &aDataManName, const Record &spec)
Create an incremental storage manager with the given name.
ISMBase & operator=(const ISMBase &that)
Assignment cannot be used.
uInt rownrSize_p
The size of a rownr in external format (local or canonical).
Definition: ISMBase.h:381
virtual void deleteManager()
The data manager will be deleted (because all its columns are requested to be deleted).
uInt cacheSize_p
The actual cache size.
Definition: ISMBase.h:365
void clearCache()
Clear the cache used by this storage manager.
uInt version() const
Get the version of the class.
Definition: ISMBase.h:387
void addBucket(rownr_t rownr, ISMBucket *bucket)
Add a bucket to the storage manager (i.e.
virtual Bool hasMultiFileSupport() const
The data manager supports use of MultiFile.
uInt cacheSize() const
Get the current cache size (in buckets).
Definition: ISMBase.h:392
void writeIndex()
Write the index (at the end of the file).
Int firstFree_p
The first free bucket.
Definition: ISMBase.h:371
BucketFile * file_p
The file containing all data.
Definition: ISMBase.h:359
ISMBucket * nextBucket(uInt &cursor, rownr_t &bucketStartRow, rownr_t &bucketNrrow)
Get the next bucket.
virtual Bool canAddColumn() const
Can the storage manager add columns? (not yet)
BucketCache * cache_p
The cache with the ISM buckets.
Definition: ISMBase.h:357
ISMBucket * getBucket(rownr_t rownr, rownr_t &bucketStartRow, rownr_t &bucketNrrow)
Get the bucket containing the given row.
ISMBase(const String &dataManagerName, uInt bucketSize, Bool checkBucketSize, uInt cacheSize)
Create an incremental storage manager with the given name.
void makeCache()
Construct the cache object (if not constructed yet).
uInt uIntSize() const
Get the size of a uInt in external format (can be canonical or local).
Definition: ISMBase.h:412
virtual void addRow64(rownr_t nrrow)
Add rows to the storage manager.
ISMBase(uInt bucketSize=0, Bool checkBucketSize=True, uInt cacheSize=1)
Create an incremental storage manager without a name.
virtual Bool canRemoveRow() const
Can the storage manager delete rows? (yes)
void showBucketLayout(ostream &os)
Show the layout of the buckets.
virtual void removeRow64(rownr_t rownr)
Delete a row from all columns.
A drop-in replacement for Block<T*>.
Definition: Block.h:814
String: the storage and methods of handling collections of characters.
Definition: String.h:225
this file contains all the compiler specific defines
Definition: mainpage.dox:28
unsigned int uInt
Definition: aipstype.h:51
int Int
Definition: aipstype.h:50
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
const Bool True
Definition: aipstype.h:43
uInt64 rownr_t
Define the type of a row number in a table.
Definition: aipsxtype.h:46