BaseTable.h

Go to the documentation of this file.

//# BaseTable.h: Abstract base class for tables
//# Copyright (C) 1994,1995,1996,1997,1998,2000,2001,2002,2003
//# Associated Universities, Inc. Washington DC, USA.
//#
//# This library is free software; you can redistribute it and/or modify it
//# under the terms of the GNU Library General Public License as published by
//# the Free Software Foundation; either version 2 of the License, or (at your
//# option) any later version.
//#
//# This library is distributed in the hope that it will be useful, but WITHOUT
//# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
//# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
//# License for more details.
//#
//# You should have received a copy of the GNU Library General Public License
//# along with this library; if not, write to the Free Software Foundation,
//# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
//#
//# Correspondence concerning AIPS++ should be addressed as follows:
//#        Internet email: casa-feedback@nrao.edu.
//#        Postal address: AIPS++ Project Office
//#                        National Radio Astronomy Observatory
//#                        520 Edgemont Road
//#                        Charlottesville, VA 22903-2475 USA
 
#ifndef TABLES_BASETABLE_H
#define TABLES_BASETABLE_H
 
 
//# Includes
#include <casacore/casa/aips.h>
#include <casacore/tables/Tables/TableInfo.h>
#include <casacore/tables/Tables/TableDesc.h>
#include <casacore/tables/Tables/StorageOption.h>
#include <casacore/casa/Utilities/Compare.h>
#include <casacore/casa/BasicSL/String.h>
#include <casacore/casa/IO/FileLocker.h>
#include <casacore/casa/Arrays/ArrayFwd.h>
#include <memory>
 
#ifdef HAVE_MPI
#include <mpi.h>
#endif
 
namespace casacore { //# NAMESPACE CASACORE - BEGIN
 
//# Forward Declarations
class RefTable;
// class TableDesc;  !Forward declaration not recognized SGI compiler
class TableLock;
class BaseColumn;
class ColumnDesc;
class TableRecord;
class Record;
class TableExprNode;
class BaseTableIterator;
class DataManager;
class IPosition;
template<class T> class Block;
template<class T> class PtrBlock;
class AipsIO;
 
 
// <summary>
// Abstract base class for tables
// </summary>
 
// <use visibility=local>
 
// <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
// </reviewed>
 
// <prerequisite>
//# Classes you should understand before using this one.
//   <li> Table
//   <li> Sort
//   <li> TableExprNode
// </prerequisite>
 
// <etymology>
// BaseTable is the (abstract) base class for different kind of tables.
// </etymology>
 
// <synopsis>
// BaseTables defines many virtual functions, which are actually
// implemented in the underlying table classes like PlainTable and
// RefTable. Other functions like sort and select are implemented
// in BaseTable itself.
//
// The functions in BaseTable and its derived classes can only be
// used by the table system classes. All user access is via the
// envelope class Table, which references (counted) BaseTable.
// </synopsis>
 
// <todo asof="$DATE:$">
//# A List of bugs, limitations, extensions or planned refinements.
//   <li> Implement function renameColumn, removeColumn.
// </todo>
 
 
class BaseTable: public std::enable_shared_from_this<BaseTable>
{
public:
 
    // Initialize the object.
    BaseTable (const String& tableName, int tableOption, rownr_t nrrow);
 
#ifdef HAVE_MPI
    // MPI version of the constructor
    BaseTable (MPI_Comm mpiComm, const String& tableName, int tableOption, rownr_t nrrow);
#endif
 
    // Common code shared by the MPI constructor and non-MPI constructor
    void BaseTableCommon (const String& tableName, int tableOption, rownr_t nrrow);
 
    // The destructor will delete the table if needed.
    virtual ~BaseTable();
 
    // Copy constructor is forbidden, because copying a table requires
    // some more knowledge (like table name of result).
    BaseTable (const BaseTable&) = delete;
 
    // Assignment is forbidden, because copying a table requires
    // some more knowledge (like table name of result).
    BaseTable& operator= (const BaseTable&) = delete;
 
    // Is the table a null table?
    // By default it is not.
    virtual Bool isNull() const;
 
    // Reopen the table for read/write.
    virtual void reopenRW() = 0;
 
    // Indicate we will only modify data values in existing tiled columns
    virtual void changeTiledDataOnly();
 
    // Is the table stored in big or little endian format?
    virtual Bool asBigEndian() const = 0;
 
    // Get the storage option used for the table.
    virtual const StorageOption& storageOption() const = 0;
 
    // Is the table in use (i.e. open) in another process?
    // If <src>checkSubTables</src> is set, it is also checked if
    // a subtable is used in another process.
    virtual Bool isMultiUsed(Bool checkSubTables) const = 0;
 
    // Get the locking info.
    virtual const TableLock& lockOptions() const = 0;
 
    // Merge the given lock info with the existing one.
    virtual void mergeLock (const TableLock& lockOptions) = 0;
 
    // Has this process the read or write lock, thus can the table
    // be read or written safely?
    virtual Bool hasLock (FileLocker::LockType) const = 0;
 
    // Try to lock the table for read or write access.
    virtual Bool lock (FileLocker::LockType, uInt nattempts) = 0;
 
    // Unlock the table. This will also synchronize the table data,
    // thus force the data to be written to disk.
    virtual void unlock() = 0;
 
    // Flush the table, i.e. write it to disk.
    virtual void flush (Bool fsync, Bool recursive) = 0;
 
    // Resync the Table object with the table file.
    virtual void resync() = 0;
 
    // Get the modify counter.
    virtual uInt getModifyCounter() const = 0;
 
    // Set the table to being changed. By default it does nothing.
    virtual void setTableChanged();
 
    // Do not write the table (used in in case of exceptions).
    void doNotWrite()
    { noWrite_p = True; }
 
    // Test if this table is writable.
    // This tells if values can be put into a column.
    virtual Bool isWritable() const = 0;
 
    // Test if the given column is writable.
    // <group>
    Bool isColumnWritable (const String& columnName) const;
    Bool isColumnWritable (uInt columnIndex) const;
    // </group>
 
    // Test if the given column is stored (otherwise it is virtual).
    // <group>
    Bool isColumnStored (const String& columnName) const;
    Bool isColumnStored (uInt columnIndex) const;
    // </group>
 
    // Get the table name.
    const String& tableName() const
    { return name_p; }
 
    // Get the names of the tables this table consists of.
    // The default implementation adds the name of this table to the block.
    virtual void getPartNames (Block<String>& names, Bool recursive) const;
 
    // Rename the table.
    // The following options can be given:
    // <dl>
    // <dt> Table::Update
    // <dd> A table with this name must already exists, which will be
    //      overwritten. When succesfully renamed, the table is unmarked
    //      for delete (if necessary).
    // <dt> Table::New
    // <dd> When a table with this name exists, it will be overwritten.
    //      When succesfully renamed, the table is unmarked
    //      for delete (if necessary).
    // <dt> Table::NewNoReplace
    // <dd> When a table with this name already exists, an exception
    //      is thrown. When succesfully renamed, the table
    //      is unmarked for delete (if necessary).
    // <dt> Table::Scratch
    // <dd> Same as Table::New, but followed by markForDelete().
    // </dl>
    // The rename function in this base class renames the table file.
    // In a derived class (e.g. PlainTable) the function should also
    // be implemented to rename subtables in its keywords.
    virtual void rename (const String& newName, int tableOption);
 
    // Copy the table and all its subtables.
    // The default implementation of deepCopy is to call copy.
    // The following options can be given:
    // <dl>
    // <dt> Table::New
    // <dd> When a table with this name exists, it will be overwritten.
    // <dt> Table::NewNoReplace
    // <dd> When a table with this name already exists, an exception
    //      is thrown.
    // <dt> Table::Scratch
    // <dd> Same as Table::New, but followed by markForDelete().
    // </dl>
    // <group>
    virtual void copy (const String& newName, int tableOption) const;
    virtual void deepCopy (const String& newName,
               const Record& dataManagerInfo,
                           const StorageOption&,
               int tableOption,
               Bool valueCopy,
               int endianFormat,
               Bool noRows) const;
    // </group>
 
    // Get the table type.
    // By default it returns Table::Plain.
    virtual int tableType() const;
 
    // Get the table option.
    int tableOption() const
    { return option_p; }
 
    // Mark the table for delete.
    // This means that the underlying table gets deleted when it is
    // actually destructed.
    // The scratchCallback function is called when needed.
    void markForDelete (Bool callback, const String& oldName);
 
    // Unmark the table for delete.
    // This means the underlying table does not get deleted when destructed.
    // The scratchCallback function is called when needed.
    void unmarkForDelete (Bool callback, const String& oldName);
 
    // Test if the table is marked for delete.
    Bool isMarkedForDelete() const
    { return delete_p; }
 
    // Get the table description.
    const TableDesc& tableDesc() const
        { return (!tdescPtr_p  ?  makeEmptyTableDesc() : *tdescPtr_p); }
 
    // Get the actual table description.
    virtual TableDesc actualTableDesc() const = 0;
 
    // Get the data manager info.
    virtual Record dataManagerInfo() const = 0;
 
    // Show the table structure (implementation of Table::showStructure).
    void showStructure (std::ostream&,
                        Bool showDataMan,
                        Bool showColumns,
                        Bool showSubTables,
                        Bool sortColumns,
                        Bool cOrder);
 
    // Get readonly access to the table keyword set.
    virtual TableRecord& keywordSet() = 0;
 
    // Get read/write access to the table keyword set.
    // This requires that the table is locked (or it gets locked
    // when using AutoLocking mode).
    virtual TableRecord& rwKeywordSet() = 0;
 
    // Get access to the TableInfo object.
    TableInfo& tableInfo()
    { return info_p; }
 
    // Get the table info of the table with the given name.
    // An empty object is returned when the table is unknown.
    static TableInfo tableInfo (const String& tableName);
 
    // Write the TableInfo object.
    virtual void flushTableInfo();
 
    // Get number of rows.
    rownr_t nrow() const
    { return nrrow_p; }
 
    // Get a column object using its index.
    virtual BaseColumn* getColumn (uInt columnIndex) const = 0;
 
    // Get a column object using its name.
    virtual BaseColumn* getColumn (const String& columnName) const = 0;
 
    // Test if it is possible to add a row to this table.
    virtual Bool canAddRow() const;
 
    // Add one or more rows and possibly initialize them.
    // This will fail for tables not supporting addition of rows.
    virtual void addRow (rownr_t nrrow = 1, Bool initialize = True);
 
    // Test if it is possible to remove a row from this table.
    virtual Bool canRemoveRow() const;
 
    // Remove rows.
    // This will fail for tables not supporting removal of rows.
    // <note role=tip>
    // The following code fragments do NOT have the same result:
    // <srcblock>
    //    tab.removeRow (10);      // remove row 10
    //    tab.removeRow (20);      // remove row 20, which was 21
    //
    //    Vector<rownr_t> vec(2);
    //    vec(0) = 10;
    //    vec(1) = 20;
    //    tab.removeRow (vec);     // remove row 10 and 20
    // </srcblock>
    // because in the first fragment removing row 10 turns the former
    // row 21 into row 20.
    // </note>
    // <group>
    virtual void removeRow (rownr_t rownr);
    void removeRow (const Vector<rownr_t>& rownrs);
    void removeRow (const Vector<uInt>& rownrs);
    // </group>
 
    // Find the data manager with the given name or for the given column.
    virtual DataManager* findDataManager (const String& name,
                                          Bool byColumn) const = 0;
 
    // Select rows using the given expression (which can be null).
    // Skip first <src>offset</src> matching rows.
    // Return at most <src>maxRow</src> matching rows.
    std::shared_ptr<BaseTable> select (const TableExprNode&,
                                       rownr_t maxRow, rownr_t offset);
 
    // Select maxRow rows and skip first offset rows. maxRow=0 means all.
    std::shared_ptr<BaseTable> select (rownr_t maxRow, rownr_t offset);
 
    // Select rows using a vector of row numbers.
    std::shared_ptr<BaseTable> select (const Vector<rownr_t>& rownrs);
 
    // Select rows using a mask block.
    // The length of the block must match the number of rows in the table.
    // If True, the corresponding row will be selected.
    std::shared_ptr<BaseTable> select (const Block<Bool>& mask);
 
    // Project the given columns (i.e. select the columns).
    std::shared_ptr<BaseTable> project (const Block<String>& columnNames);
 
    // Do logical operations on a table.
    // <group>
    // intersection with another table
    std::shared_ptr<BaseTable> tabAnd (BaseTable*);
    // union with another table
    std::shared_ptr<BaseTable> tabOr  (BaseTable*);
    // subtract another table
    std::shared_ptr<BaseTable> tabSub (BaseTable*);
    // xor with another table
    std::shared_ptr<BaseTable> tabXor (BaseTable*);
    // take complement
    std::shared_ptr<BaseTable> tabNot ();
    // </group>
 
    // Sort a table on one or more columns of scalars.
    std::shared_ptr<BaseTable> sort
    (const Block<String>& columnNames,
     const Block<std::shared_ptr<BaseCompare>>& compareObjects,
     const Block<Int>& sortOrder, int sortOption,
     std::shared_ptr<Vector<rownr_t>> sortIterBoundaries = nullptr,
     std::shared_ptr<Vector<size_t>> sortIterKeyIdxChange = nullptr);
 
    // Create an iterator.
    BaseTableIterator* makeIterator (const Block<String>& columnNames,
                                     const Block<std::shared_ptr<BaseCompare>>&,
                                     const Block<Int>& orders, int option,
                                     bool cacheIterationBoundaries = false);
 
    // Add one or more columns to the table.
    // The default implementation throws an "invalid operation" exception.
    // <group>
    virtual void addColumn (const ColumnDesc& columnDesc, Bool addToParent);
    virtual void addColumn (const ColumnDesc& columnDesc,
                const String& dataManager, Bool byName,
                            Bool addToParent);
    virtual void addColumn (const ColumnDesc& columnDesc,
                const DataManager& dataManager, Bool addToParent);
    virtual void addColumn (const TableDesc& tableDesc,
                const DataManager& dataManager, Bool addToParent);
    // </group>
 
    // Add one or more columns to the table.
    // The data manager to use is described in the record.
    void addColumns (const TableDesc& tableDesc, const Record& dmInfo,
                     Bool addToParent);
 
    // Test if columns can be removed.
    virtual Bool canRemoveColumn (const Vector<String>& columnNames) const = 0;
 
    // Remove columns.
    virtual void removeColumn (const Vector<String>& columnNames) = 0;
 
    // Check if the set of columns can be removed.
    // It checks if columns have not been specified twice and it
    // checks if they exist.
    // If the flag is set an exception is thrown if errors are found.
    Bool checkRemoveColumn (const Vector<String>& columnNames,
                Bool throwException) const;
 
    // Test if a column can be renamed.
    virtual Bool canRenameColumn (const String& columnName) const = 0;
 
    // Rename a column.
    virtual void renameColumn (const String& newName,
                   const String& oldName) = 0;
 
    // Rename a hypercolumn.
    virtual void renameHypercolumn (const String& newName,
                    const String& oldName) = 0;
 
    // Get a vector of row numbers.
    // By default it returns the row numbers 0..nrrow()-1.
    // It needs to be implemented for RefTable only.
    virtual Vector<rownr_t> rowNumbers() const;
 
    // Get pointer to root table (i.e. parent of a RefTable).
    // Default it is this table.
    // It is meant for the reference tables after a select or sort which
    // can then still name their parent as the root.
    virtual BaseTable* root();
 
    // Tell if the table is in row order.
    // By default it is, since normally a table is always in row order.
    // It is meant for RefTable-s, where the rows can be in
    // another (sorted) order.
    virtual Bool rowOrder() const;
 
    // By the default the table cannot return the storage of rownrs.
    // That can only be done by a RefTable, where it is implemented.
    virtual Vector<rownr_t>& rowStorage();
 
    // Adjust the row numbers to be the actual row numbers in the
    // root table. This is, for instance, used when a RefTable is sorted.
    // Optionally it also determines if the resulting rows are in order.
    virtual Bool adjustRownrs (rownr_t nrrow, Vector<rownr_t>& rownrs,
                   Bool determineOrder) const;
 
    // Do the actual sort.
    // The default implementation is suitable for almost all cases.
    // Only in RefTable a smarter implementation is provided.
    virtual std::shared_ptr<BaseTable> doSort
    (PtrBlock<BaseColumn*>&,
     const Block<std::shared_ptr<BaseCompare>>&,
     const Block<Int>& sortOrder,
     int sortOption,
     std::shared_ptr<Vector<rownr_t>> sortIterBoundaries,
     std::shared_ptr<Vector<size_t>> sortIterKeyIdxChange);
 
    // Create a RefTable object.
    std::shared_ptr<RefTable> makeRefTable (Bool rowOrder,
                                            rownr_t initialNrrow);
 
    // Check if the row number is valid.
    // It throws an exception if out of range.
    void checkRowNumber (rownr_t rownr) const
        { if (rownr >= nrrow_p + nrrowToAdd_p) checkRowNumberThrow (rownr); }
 
    // Get the table's trace-id.
    int traceId() const
        { return itsTraceId; }
 
protected:
    std::weak_ptr<BaseTable> thisPtr_p; //# pointer to itself (to make shared_ptr)
    rownr_t        nrrow_p;             //# #rows in this table
    rownr_t        nrrowToAdd_p;        //# #rows to be added
    std::shared_ptr<TableDesc> tdescPtr_p;   //# Pointer to table description
    String         name_p;              //# table name
    int            option_p;            //# Table constructor option
    Bool           noWrite_p;           //# False = do not write the table
    Bool           delete_p;            //# True = delete when destructed
    TableInfo      info_p;              //# Table information (type, etc.)
    Bool           madeDir_p;           //# True = table dir has been created
    int            itsTraceId;          //# table-id for TableTrace tracing
 
    // Do the callback for scratch tables (if callback is set).
    void scratchCallback (Bool isScratch, const String& oldName) const;
 
    // Create the table directory when needed (and possible).
    // When the file already exists, check if it is a directory.
    // It returns True when it actually created the directory.
    Bool makeTableDir();
 
    // Make a true deep copy of the table.
    // The table is flushed before making the copy.
    void trueDeepCopy (const String& newName,
               const Record& dataManagerInfo,
                       const StorageOption&,
               int tableOption,
               int endianFormat,
               Bool noRows) const;
 
    // Prepare for copying or renaming a table.
    // It checks if the target table already exists and removes it
    // when necessary.
    void prepareCopyRename (const String& newName, int tableOption) const;
 
    // Rename the subtables (used by rename function).
    virtual void renameSubTables (const String& newName,
                  const String& oldName);
 
    // Check if the table already exists.
    // Throw an exception if so.
    void throwIfTableExists();
 
    // Test if the table is opened for write.
    Bool openedForWrite() const;
 
    // Start writing a table. It does a putstart and writes <src>nrrow_p</src>.
    // It should be ended by calling <src>writeEnd</src>.
    void writeStart (AipsIO&, Bool bigEndian);
 
    // End writing a table.
    void writeEnd (AipsIO&);
 
    // Should the table be written.
    // This flag is False if an exception was thrown.
    Bool shouldNotWrite() const
    { return noWrite_p; }
 
    // Read the TableInfo object.
    void getTableInfo();
 
private:
    // Show a possible extra table structure header.
    // It is used by e.g. RefTable to show which table is referenced.
    virtual void showStructureExtra (std::ostream&) const;
 
    // Show the info of the given columns.
    // Sort the columns if needed.
    void showColumnInfo (ostream& os, const TableDesc&, uInt maxNameLength,
                         const Array<String>& columnNames, Bool sort,
                         Bool cOrder) const;
 
    // Throw an exception for checkRowNumber.
    void checkRowNumberThrow (rownr_t rownr) const;
 
    // Check if the tables combined in a logical operation have the
    // same root.
    void logicCheck (BaseTable* that);
 
    // Get the rownrs of the table in ascending order to be
    // used in the logical operation on the table.
    Vector<rownr_t> logicRows();
 
    // Make an empty table description.
    // This is used if one asks for the description of a NullTable.
    // Creating an empty TableDesc in the NullTable takes too much time.
    // Furthermore it causes static initialization order problems.
    const TableDesc& makeEmptyTableDesc() const;
 
    // Make the name absolute.
    // It first checks if the name contains valid characters (not only . and /).
    String makeAbsoluteName (const String& name) const;
 
#ifdef HAVE_MPI
    // MPI communicator for parallel I/O.
    // When using an MPI-disabled casacore, MPI applications have always been
    // able to create Tables from each rank independently. Defaulting this
    // communicator to MPI_COMM_SELF preserves that expectation.
    MPI_Comm itsMpiComm = MPI_COMM_SELF;
#endif
};
 
 
 
 
} //# NAMESPACE CASACORE - END
 
#endif