table_io.cpp

Go to the documentation of this file.

//                                                       //
//                         SAGA                          //
//                                                       //
//      System for Automated Geoscientific Analyses      //
//                                                       //
//           Application Programming Interface           //
//                                                       //
//                  Library: SAGA_API                    //
//                                                       //
//-------------------------------------------------------//
//                                                       //
//                    table_io.cpp                       //
//                                                       //
//          Copyright (C) 2005 by Olaf Conrad            //
//                                                       //
//-------------------------------------------------------//
//                                                       //
// This file is part of 'SAGA - System for Automated     //
// Geoscientific Analyses'.                              //
//                                                       //
// This library is free software; you can redistribute   //
// it and/or modify it under the terms of the GNU Lesser //
// General Public License as published by the Free       //
// Software Foundation, either version 2.1 of the        //
// License, or (at your option) any later version.       //
//                                                       //
// This library is distributed in the hope that it will  //
// be useful, but WITHOUT ANY WARRANTY; without even the //
// implied warranty of MERCHANTABILITY or FITNESS FOR A  //
// PARTICULAR PURPOSE. See the GNU Lesser General Public //
// License for more details.                             //
//                                                       //
// You should have received a copy of the GNU Lesser     //
// General Public License along with this program; if    //
// not, see <http://www.gnu.org/licenses/>.              //
//                                                       //
//-------------------------------------------------------//
//                                                       //
//    contact:    Olaf Conrad                            //
//                Institute of Geography                 //
//                University of Goettingen               //
//                Goldschmidtstr. 5                      //
//                37077 Goettingen                       //
//                Germany                                //
//                                                       //
//    e-mail:     oconrad@saga-gis.org                   //
//                                                       //
 
//---------------------------------------------------------
#include "table.h"
#include "table_dbase.h"
 

//                                                       //
//                                                       //
//                                                       //
 
//---------------------------------------------------------
bool CSG_Table::On_Reload(void)
{
        return( Create(Get_File_Name(false)) );
}
 
//---------------------------------------------------------
bool CSG_Table::On_Delete(void)
{
        return( SG_File_Delete(Get_File_Name(false)) );
}
 

//                                                       //
 
//---------------------------------------------------------
bool CSG_Table::Set_File_Encoding(int Encoding)
{
        if( Encoding >= 0 && Encoding < SG_FILE_ENCODING_UNDEFINED )
        {
                m_Encoding = Encoding;
 
                return( true );
        }
 
        return( false );
}
 

//                                                       //
 
//---------------------------------------------------------
bool CSG_Table::Load(const CSG_String &File, int Format, SG_Char Separator, int Encoding)
{
        Set_File_Encoding(Encoding);
 
        if( !SG_File_Exists(File) )
        {
                return( false );
        }
 
        //-----------------------------------------------------
        if( Format == TABLE_FILETYPE_Undefined )
        {
                Format = SG_File_Cmp_Extension(File, "dbf") ? TABLE_FILETYPE_DBase : TABLE_FILETYPE_Text;
        }
 
        //-----------------------------------------------------
        Destroy();
 
        switch( Format )
        {
        case TABLE_FILETYPE_Text:   default: if( !_Load_Text (File, true , Separator) ) return( false ); break;
        case TABLE_FILETYPE_Text_NoHeadLine: if( !_Load_Text (File, false, Separator) ) return( false ); break;
        case TABLE_FILETYPE_DBase          : if( !_Load_DBase(File                  ) ) return( false ); break;
        }
 
        //-----------------------------------------------------
        Set_Name(SG_File_Get_Name(File, false));
 
        Load_MetaData(File);
 
        CSG_MetaData *pFields = Get_MetaData_DB().Get_Child("FIELDS");
 
        if( pFields && pFields->Get_Children_Count() == Get_Field_Count() )
        {
                for(int Field=0; Field<Get_Field_Count(); Field++)
                {
                        Set_Field_Name(Field, pFields->Get_Content(Field));
                }
        }
 
        //-----------------------------------------------------
        return( true );
}
 
//---------------------------------------------------------
bool CSG_Table::Save(const CSG_String &File, int Format)
{
        return( Save(File, Format, '\0', m_Encoding) );
}
 
//---------------------------------------------------------
bool CSG_Table::Save(const CSG_String &File, int Format, SG_Char Separator, int Encoding, bool bMetaData)
{
        if( File.is_Empty() )
        {
                return( *Get_File_Name(false) ? Save(Get_File_Name(false), Format) : false );
        }
 
        SG_UI_Msg_Add(CSG_String::Format("%s %s: %s...", _TL("Saving"), _TL("table"), File.c_str()), true);
 
        Set_File_Encoding(Encoding);
 
        //-----------------------------------------------------
        if( Format <= TABLE_FILETYPE_Undefined || Format > TABLE_FILETYPE_DBase )
        {
                if( SG_File_Cmp_Extension(File, "dbf") )
                {
                        Format = TABLE_FILETYPE_DBase;
                }
                else
                {
                        Format = TABLE_FILETYPE_Text;
 
                        if( Separator == '\0' )
                        {
                                Separator = SG_File_Cmp_Extension(File, "csv") ? ',' : '\t'; // comma separated values or tab spaced text
                        }
                }
        }
 
        //-----------------------------------------------------
        bool bResult = false;
 
        switch( Format )
        {
        case TABLE_FILETYPE_Text:   default: bResult = _Save_Text (File, true , Separator); break;
        case TABLE_FILETYPE_Text_NoHeadLine: bResult = _Save_Text (File, false, Separator); break;
        case TABLE_FILETYPE_DBase          : bResult = _Save_DBase(File                  ); break;
        }
 
        //-----------------------------------------------------
        CSG_MetaData *pFields = Get_MetaData_DB().Get_Child("FIELDS");
 
        if( !pFields )
        {
                pFields = Get_MetaData_DB().Add_Child("FIELDS");
        }
 
        pFields->Del_Children();
 
        for(int Field=0; Field<Get_Field_Count(); Field++)
        {
                pFields->Add_Child("FIELD", Get_Field_Name(Field))->Add_Property("TYPE", gSG_Data_Type_Identifier[Get_Field_Type(Field)]);
        }
 
        //-----------------------------------------------------
        if( bResult )
        {
                Set_Modified(false);
 
                Set_Update_Flag();
 
                Set_File_Type(Format);
 
                Set_File_Name(File, true);
 
                if( bMetaData )
                {
                        Save_MetaData(File);
                }
 
                SG_UI_Msg_Add(_TL("okay"), false, SG_UI_MSG_STYLE_SUCCESS);
 
                return( true );
        }
 
        SG_UI_Msg_Add(_TL("failed"), false, SG_UI_MSG_STYLE_FAILURE);
 
        return( false );
}
 

//                                                       //
//                         Text                          //
//                                                       //
 
//---------------------------------------------------------
size_t CSG_Table::_Load_Text_Trim(CSG_String &s, const SG_Char Separator)
{
        for(size_t i=0; i<s.Length(); i++)
        {
                SG_Char c = s[i];
 
                if( c == Separator || (c != ' ' && c != '\t' && c != '\n' && c != '\v' && c != '\f' && c != '\r') )
                {
                        if( i > 0 )
                        {
                                s = s.Right(s.Length() - i);
                        }
 
                        return( i );
                }
        }
 
        return( 0 );
}
 
//---------------------------------------------------------
size_t  CSG_Table::_Load_Text_EndQuote(const CSG_String &s, const SG_Char Separator)
{
        if( s.Length() > 1 && s[0] == '\"' )
        {
                bool bInQuotes = true;
 
                for(size_t i=1; i<s.Length(); i++)
                {
                        if( bInQuotes )
                        {
                                if( s[i] == '\"' )
                                {
                                        bInQuotes = false;
                                }
                        }
                        else if( s[i] == '\"' )
                        {
                                bInQuotes = true;
                        }
                        else if( s[i] == Separator )
                        {
                                return( i );
                        }
                }
 
                if( s[s.Length() - 1] == '\"' )
                {
                        return( s.Length() );
                }
        }
 
        return( 0 );
}
 
//---------------------------------------------------------
bool CSG_Table::_Load_Text(const CSG_String &File, bool bHeadline, const SG_Char _Separator)
{
        CSG_File Stream;
 
        if( Stream.Open(File, SG_FILE_R, false, m_Encoding) == false )
        {
                return( false );
        }
 
        sLong fLength = Stream.Length();
 
        if( fLength < 1 )
        {
                return( false );
        }
 
        //-----------------------------------------------------
        CSG_String Line;
 
        if( !Stream.Read_Line(Line) ) // end-of-file !
        {
                return( false );
        }
 
        if( Line[0] == 65279 ) // 65279 => '\uFEFF' => BOM => zero-width no-break space ! ...to do: recognizing other 'magic' first characters !
        {
                Line.Remove(0, 1);
        }
 
        bool bCSV = SG_File_Cmp_Extension(File, "csv");
 
        while( Line.is_Empty() || (bCSV && Line[0] == '#') ) // empty or comment
        {
                if( !Stream.Read_Line(Line) ) // end-of-file !
                {
                        return( false );
                }
        }
 
        //-----------------------------------------------------
        SG_Char Separator = _Separator;
 
        if( Separator == '\0' )
        {
                if( bCSV ) // comma separated values
                {
                        Separator = Line.Find(';') >= 0 ? ';' : ','; // assume semicolon as value separator, comma as decimal separator!
                }
                else // assume tab spaced text table
                {
                        Separator = '\t';
                }
        }
 
        bool bComma2Point = bCSV && Separator == ';';
 
        //-----------------------------------------------------
        CSG_Table Table;
 
        _Load_Text_Trim(Line, Separator);
 
        while( !Line.is_Empty() )
        {
                CSG_String Value;
 
                if( Line[0] == '\"' ) // value in quotas
                {
                        Value = Line.AfterFirst('\"').BeforeFirst('\"');
                        Line  = Line.AfterFirst('\"').AfterFirst ('\"');
                }
                else
                {
                        Value = Line.BeforeFirst(Separator);
                }
 
                Line = Line.AfterFirst(Separator); _Load_Text_Trim(Line, Separator);
 
                if( !bHeadline || Value.Length() == 0 )
                {
                        Value.Printf("F%02d", Table.Get_Field_Count() + 1);
                }
 
                Table.Add_Field(Value, SG_DATATYPE_String);
        }
 
        //-----------------------------------------------------
        TSG_Data_Type *Types = new TSG_Data_Type[Table.Get_Field_Count()];
 
        for(int Field=0; Field<Table.Get_Field_Count(); Field++)
        {
                Types[Field] = SG_DATATYPE_Int;
        }
 
        if( !bHeadline )
        {
                Stream.Seek_Start();
        }
 
        while( Stream.Read_Line(Line) && SG_UI_Process_Set_Progress((double)Stream.Tell(), (double)fLength) )
        {
                if( Line.is_Empty() || (bCSV && Line[0] == '#') ) // empty or comment
                {
                        continue;
                }
 
                CSG_Table_Record &Record = *Table.Add_Record();
 
                _Load_Text_Trim(Line, Separator);
 
                for(int Field=0; Field<Table.Get_Field_Count() && !Line.is_Empty(); Field++)
                {
                        size_t Position = _Load_Text_EndQuote(Line, Separator); CSG_String Value;
 
                        if( Position > 0 ) // value in quotas !!!
                        {
                                if( Position - 2 > 0 )
                                {
                                        Value = Line.Mid(1, Position - 2);
                                }
                                else
                                {
                                        Value.Clear();
                                }
 
                                Line = Line.Right(Line.Length() - Position);
 
                                Types[Field] = SG_DATATYPE_String;
                        }
                        else
                        {
                                Value = Line.BeforeFirst(Separator);
 
                                if( bComma2Point )
                                {
                                        Value.Replace(",", ".");
                                }
                        }
 
                        Line = Line.AfterFirst(Separator); _Load_Text_Trim(Line, Separator);
 
                        //---------------------------------------------
                        if( Types[Field] != SG_DATATYPE_String && !Value.is_Empty() )
                        {
                                if( Value[0] == '0' && Value[1] != '.' && Value[1] != '\0' ) // keep leading zero(s) => don't interpret as number !
                                {
                                        Types[Field] = SG_DATATYPE_String;
                                }
                                else try
                                {
                                        size_t pos; double number = std::stod(Value.to_StdString(), &pos);
 
                                        if( pos < Value.Length() )
                                        {
                                                Types[Field] = SG_DATATYPE_String;
                                        }
                                        else if( Types[Field] != SG_DATATYPE_Double && (number - (int)number != 0. || Value.Find('.') >= 0) )
                                        {
                                                Types[Field] = SG_DATATYPE_Double;
                                        }
                                }
                                catch(...)
                                {
                                        Types[Field] = SG_DATATYPE_String;
                                }
                        }
 
                        Record.Set_Value(Field, Value);
                }
        }
 
        //-----------------------------------------------------
        if( Table.Get_Field_Count() > 0 )
        {
                for(int Field=0; Field<Table.Get_Field_Count(); Field++)
                {
                        Add_Field(Table.Get_Field_Name(Field), Types[Field]);
                }
 
                for(sLong i=0; i<Table.Get_Count() && SG_UI_Process_Set_Progress(i, Table.Get_Count()); i++)
                {
                        CSG_Table_Record &Record = *Add_Record();
 
                        for(int Field=0; Field<Get_Field_Count(); Field++)
                        {
                                if( *Table[i].asString(Field) )
                                {
                                        Record.Set_Value(Field, Table[i].asString(Field));
                                }
                                else
                                {
                                        Record.Set_NoData(Field);
                                }
                        }
                }
        }
 
        delete[](Types);
 
        SG_UI_Process_Set_Ready();
 
        return( Get_Field_Count() > 0 );
}
 
//---------------------------------------------------------
bool CSG_Table::_Save_Text(const CSG_String &File, bool bHeadline, const SG_Char Separator)
{
        CSG_File Stream;
 
        if( Get_Field_Count() <= 0 || Stream.Open(File, SG_FILE_W, false, m_Encoding) == false )
        {
                return( false );
        }
 
        //-----------------------------------------------------
        for(int Field=0; Field<Get_Field_Count(); Field++)
        {
                Stream.Printf("%s%c", Get_Field_Name(Field), Field < Get_Field_Count() - 1 ? Separator : '\n');
        }
 
        //-----------------------------------------------------
        for(sLong i=0; i<Get_Count() && SG_UI_Process_Set_Progress(i, Get_Count()); i++)
        {
                CSG_Table_Record &Record = *Get_Record_byIndex(i);
 
                for(int Field=0; Field<Get_Field_Count(); Field++)
                {
                        CSG_String Value;
 
                        if( !Record.is_NoData(Field) )
                        {
                                Value = Record.asString(Field);
                        }
 
                        switch( Get_Field_Type(Field) )
                        {
                        default:
                                break;
 
                        case SG_DATATYPE_String:
                        case SG_DATATYPE_Date  :
                                Value.Replace("\t", " ");
                                Value.Replace("\n", " ");
                                Value = "\"" + Value + "\"";
                                break;
                        }
 
                        Stream.Write(Value + (Field < Get_Field_Count() - 1 ? Separator : SG_T('\n')));
                }
        }
 
        //-----------------------------------------------------
        SG_UI_Process_Set_Ready();
 
        return( true );
}
 

//                                                       //
//                        DBase                          //
//                                                       //
 
//---------------------------------------------------------
bool CSG_Table::_Load_DBase(const CSG_String &File)
{
        CSG_Table_DBase dbf(m_Encoding);
 
        return( dbf.Open_Read(File, this) );
}
 
//---------------------------------------------------------
bool CSG_Table::_Save_DBase(const CSG_String &File)
{
        CSG_Table_DBase dbf(m_Encoding);
 
        return( dbf.Open_Write(File, this) );
}
 

//                                                       //
//                     From/To Text                      //
//                                                       //
 
//---------------------------------------------------------
bool CSG_Table::from_Text(const CSG_String &Text)
{
        if( Text.is_Empty() )
        {
                return( false );
        }
 
        Destroy(); Set_Name(_TL("New Table"));
 
        //-----------------------------------------------------
        CSG_Strings Values, Records(SG_String_Tokenize(Text, "\r\n"));
 
        Values = (SG_String_Tokenize(Records[0], "\t"));
 
        TSG_Data_Type *Types = new TSG_Data_Type[Values.Get_Count()];
 
        for(int Field=0; Field<Values.Get_Count(); Field++)
        {
                Add_Field(Values[Field], SG_DATATYPE_String); Types[Field] = SG_DATATYPE_Int;
        }
 
        //-----------------------------------------------------
        for(int i=1; i<Records.Get_Count(); i++)
        {
                CSG_Table_Record &Record = *Add_Record();
 
                Values = (SG_String_Tokenize(Records[i], "\t"));
 
                for(int Field=0; Field<Values.Get_Count() && Field<Get_Field_Count(); Field++)
                {
                        CSG_String Value(Values[Field]);
 
                        if( Value.Length() >= 2 && Value[0] == '\"' && Value[Value.Length() - 1] == '\"' ) // is string, remove quota
                        {
                                Types[Field] = SG_DATATYPE_String; Value = Value.Mid(1, Value.Length() - 2);
                        }
 
                        Record.Set_Value(Field, Value);
 
                        if( Types[Field] != SG_DATATYPE_String && !Value.is_Empty() )
                        {
                                if( Value[0] == '0' && Value[1] != '.' ) // keep leading zero(s) => don't interpret as number !
                                {
                                        Types[Field] = SG_DATATYPE_String;
                                }
                                else try
                                {
                                        size_t pos; double number = std::stod(Value.to_StdString(), &pos);
 
                                        if( pos < Value.Length() )
                                        {
                                                Types[Field] = SG_DATATYPE_String;
                                        }
                                        else if( Types[Field] != SG_DATATYPE_Double && (number - (int)number != 0. || Value.Find('.') >= 0) )
                                        {
                                                Types[Field] = SG_DATATYPE_Double;
                                        }
                                }
                                catch(...)
                                {
                                        Types[Field] = SG_DATATYPE_String;
                                }
                        }
                }
        }
 
        //-----------------------------------------------------
        for(int Field=0; Field<Get_Field_Count(); Field++)
        {
                Set_Field_Type(Field, Types[Field]);
        }
 
        delete[](Types);
 
        return( Get_Field_Count() > 1 || Get_Count() > 0 );
}
 
//---------------------------------------------------------
CSG_String CSG_Table::to_Text(bool Selection, bool Header, bool QuoteStrings) const
{
        CSG_String Text;
 
        if( Header )
        {
                for(int Field=0; Field<Get_Field_Count(); Field++)
                {
                        Text += Get_Field_Name(Field); Text += 1 + Field < Get_Field_Count() ? '\t' : '\n';
                }
        }
 
        sLong n = Selection ? Get_Selection_Count() : Get_Count();
 
        for(sLong i=0; i<n; i++)
        {
                CSG_Table_Record &Record = Selection ? *Get_Selection(i) : *Get_Record_byIndex(i);
 
                for(int Field=0; Field<Get_Field_Count(); Field++)
                {
                        CSG_String Value(Record.asString(Field));
 
                        if( Get_Field_Type(Field) == SG_DATATYPE_String )
                        {
                                Value.Replace("\t", " "); Value.Replace("\n", " ");
 
                                if( QuoteStrings )
                                {
                                        Value = "\"" + Value + "\"";
                                }
                        }
 
                        Text += Value + (1 + Field < Get_Field_Count() ? '\t' : '\n');
                }
        }
 
        return( Text );
}
 

//                                                       //
//                      Serialize                        //
//                                                       //
 
//---------------------------------------------------------
bool CSG_Table::Serialize(CSG_File &Stream, bool bSave)
{
        const SG_Char Separator = SG_T('\t');
 
        //-----------------------------------------------------
        if( bSave )
        {
                Stream.Printf("%d %d\n", m_nFields, m_nRecords);
 
                for(int Field=0; Field<m_nFields; Field++)
                {
                        Stream.Printf("%d \"%s\"\n", Get_Field_Type(Field), Get_Field_Name(Field));
                }
 
                for(sLong i=0; i<m_nRecords; i++)
                {
                        for(int Field=0; Field<m_nFields; Field++)
                        {
                                Stream.Printf("%s%c", Get_Record(i)->asString(Field), Field < m_nFields - 1 ? Separator : '\n');
                        }
                }
 
                return( true );
        }
 
        //-----------------------------------------------------
        CSG_String sLine; int nFields, FieldType; sLong nRecords;
 
        if( Stream.Read_Line(sLine) && SG_SSCANF(sLine, SG_T("%d %lld"), &nFields, &nRecords) == 2 && nFields > 0 )
        {
                Destroy();
 
                for(int Field=0; Field<nFields; Field++)
                {
                        if( Stream.Read_Line(sLine) && SG_SSCANF(sLine, SG_T("%d"), &FieldType) == 1 )
                        {
                                Add_Field(sLine.AfterFirst('\"').BeforeFirst('\"'), (TSG_Data_Type)FieldType);
                        }
                }
 
                for(sLong i=0; i<nRecords; i++)
                {
                        if( Stream.Read_Line(sLine) )
                        {
                                CSG_Table_Record &Record = *Add_Record();
 
                                for(int Field=0; Field<m_nFields; Field++)
                                {
                                        Record.Set_Value(Field, sLine.BeforeFirst(Separator));
 
                                        sLine = sLine.AfterFirst(Separator);
                                }
                        }
                }
 
                return( true );
        }
 
        //-----------------------------------------------------
        return( false );
}
 

//                                                       //
//                                                       //
//                                                       //
 
//---------------------------------------------------------