////////////////////////////////////////////////////////////////////////////////
// Copyright (c) 2001-2026 Alexey Kuryakin daqgroup@mail.ru under MIT license //
////////////////////////////////////////////////////////////////////////////////

////////////////////////////////////////////////////////////////////////////////
// This file is part of the CRW-DAQ project by DaqGroup - component CRWLIB.   //
////////////////////////////////////////////////////////////////////////////////

////////////////////////////////////////////////////////////////////////////////
// Purpose:                                                                   //
// Text file [MetaData] section support uses for file integrity checking.     //
////////////////////////////////////////////////////////////////////////////////

////////////////////////////////////////////////////////////////////////////////
// History:                                                                   //
// 20170221 - Creation                                                        //
// 20170222 - First release                                                   //
// 20170222 - @Checksum = MD5 in hex                                          //
// 20170412 - FileAddMetaData,FileCheckMetaData redesign                      //
// 20170412 - small FileAddMetaData corrections                               //
// 20230523 - Modified for FPC (A.K.)                                         //
// 20240623 - UnifyEol: fix CRLF/LF difference of checksum                    //
////////////////////////////////////////////////////////////////////////////////

unit _crw_meta; // MetaData for text (config) files.

{$I _crw_sysdef.inc}

{$I _crw_sysmode.inc}

{$WARN 5023 off : Unit "$1" not used in $2}

interface

uses
 //////////////////////////////////////////////////////
 {$I _crw_uses_first.inc} // NB: MUST BE FIRST USES !!!
 //////////////////////////////////////////////////////
 sysutils, classes, math,
 _crw_alloc, _crw_str, _crw_rtc, _crw_fio, _crw_crypt;

 ////////////////////////////////////////////////////////////////////////////////////////
 // [MetaData] section contains some information on text, like @CheckSum, @TimeStamp etc.
 // [MetaData] section should be ignored by configuration analyser, because it's not part
 // of the original text. [MetaData] section cuts from text when @CheckSum calculated, so
 // any data can be added to [MetaData] without changing @CheckSum value. But any changes
 // of text outside [MetaData] ... [] will cause TextCheckMetaData(..) to be mde_Invalid.
 ////////////////////////////////////////////////////////////////////////////////////////
const
 MetaDataSectionName = '[MetaData]'; // MetaData section name
 EmptySectionName    = '[]';         // Empty section name uses as section separator
 mde_Success         = 0;            // MetaData error code: Success, text/file checksum OK
 mde_Missing         = 1;            // MetaData error code: [MetaData] file/section missing
 mde_Invalid         = 2;            // MetaData error code: [MetaData] @CheckSum is invalid
 mde_Failure         = 3;            // MetaData error code: [MetaData] failure

function TextAddMetaData(const Lines:TStringList; const aMetaText:LongString=''):TStringList; overload;
function TextAddMetaData(const aText:LongString; const aMetaText:LongString=''):LongString; overload;
function TextGetMetaData(const Lines:TStringList):LongString; overload;
function TextGetMetaData(const aText:LongString):LongString; overload;
function TextCheckMetaData(const aText:LongString):Integer;
function FileAddMetaData(const aFile:LongString; const aMetaText:LongString=''):Integer;
function FileAddMetaDataList(const aFile:LongString; const aMetaText:LongString=''):Integer;
function FileCheckMetaData(const aFile:LongString; Notify:Boolean=false):Integer;
function FileCheckMetaDataList(const aFile:LongString; Notify:Boolean=false):Integer;
function TextDropMetaData(const Lines:TStringList):TStringList; overload;
function mde_Message(mde:Integer):LongString;

function TextMetaDataUtility(Arg1,Arg2,Arg3:LongString):Integer;

type ETextMetaData = class(ESoftException);

implementation

procedure FatalBug(const Msg:AnsiString);
begin
 RAISE ETextMetaData.Create(Msg);
end;

function UnifyEol(const aText:LongString):LongString;
begin
 if (aText<>'')
 then Result:=ValidateCRLF(aText)
 else Result:='';
end;

function TextDropMetaData(const Lines:TStringList):TStringList; overload;
var i:Integer; Sect:LongString; IsMetaData:Boolean;
begin
 Result:=Lines;
 if Assigned(Lines) then
 try
  i:=0; IsMetaData:=false; Sect:='';
  while i<Lines.Count do begin
   if ExtractSectionTitle(Lines[i],Sect) then begin
    if IsMetaData and SameText(Sect,EmptySectionName) then begin
     IsMetaData:=false;
     Lines.Delete(i);
     Continue;
    end;
    IsMetaData:=SameText(Sect,MetaDataSectionName);
   end;
   if IsMetaData then begin
    Lines.Delete(i);
    Continue;
   end;
   inc(i);
  end;
 except
  on E:Exception do BugReport(E,nil,'TextDropMetaData');
 end;
end;

function TextDropMetaData(const aText:LongString):LongString; overload;
var Lines:TStringList;
begin
 Result:='';
 if Length(aText)>0 then
 try
  Lines:=TStringList.Create;
  try
   Lines.Text:=AdjustLineBreaks(aText);
   TextDropMetaData(Lines);
   Result:=Lines.Text;
  finally
   Kill(Lines);
  end;
 except
  on E:Exception do BugReport(E,nil,'TextDropMetaData');
 end;
end;

function TextGetMetaData(const Lines:TStringList):LongString; overload;
var i:Integer; Sect:LongString; IsMetaData:Boolean; Temp:TStringList;
begin
 Result:='';
 if Assigned(Lines) then
 try
  Temp:=TStringList.Create;
  try
   i:=0; IsMetaData:=false; sect:='';
   while i<Lines.Count do begin
    if ExtractSectionTitle(Lines[i],Sect) then begin
     IsMetaData:=SameText(Sect,MetaDataSectionName);
    end else begin
     if IsMetaData then Temp.Add(Lines[i]);
    end;
    inc(i);
   end;
   Result:=Temp.Text;
  finally
   Kill(Temp);
  end;
 except
  on E:Exception do BugReport(E,nil,'TextGetMetaData');
 end;
end;

function TextGetMetaData(const aText:LongString):LongString; overload;
var Lines:TStringList;
begin
 Result:='';
 if Length(aText)>0 then
 try
  Lines:=TStringList.Create;
  try
   Lines.Text:=AdjustLineBreaks(aText);
   Result:=TextGetMetaData(Lines);
  finally
   Kill(Lines);
  end;
 except
  on E:Exception do BugReport(E,nil,'TextGetMetaData');
 end;
end;

function TextAddMetaData(const Lines:TStringList; const aMetaText:LongString=''):TStringList; overload;
var ms:Double; dt,cs,TheText:LongString; Meta:TStringList;
begin
 Result:=Lines;
 if Assigned(Lines) then
 try
  ms:=mSecNow;
  TextDropMetaData(Lines);
  TheText:=Lines.Text;
  cs:=Hex_Encode(GetMD5FromText(UnifyEol(TheText),df_Bin));
  dt:=GetDateStr(ms,'.',true)+'-'+GetTimeStr(ms,':');
  Lines.Add(MetaDataSectionName);
  Lines.Add(Format('@CheckSum=%s',[cs]));
  Lines.Add(Format('@DateTime=%s',[dt]));
  Lines.Add(Format('@TimeStamp=%g',[ms]));
  if Length(aMetaText)>0 then begin
   Meta:=TStringList.Create;
   try
    Meta.Text:=AdjustLineBreaks(aMetaText);
    Lines.Text:=Lines.Text+Meta.Text;
   finally
    Kill(Meta);
   end;
  end;
  Lines.Add(EmptySectionName);
 except
  on E:Exception do BugReport(E,nil,'TextAddMetaData');
 end;
end;

function TextAddMetaData(const aText:LongString; const aMetaText:LongString=''):LongString; overload;
var Lines:TStringList;
begin
 Result:='';
 try
  Result:=aText;
  Lines:=TStringList.Create;
  try
   Lines.Text:=AdjustLineBreaks(aText);
   TextAddMetaData(Lines,aMetaText);
   Result:=Lines.Text;
  finally
   Kill(Lines);
  end;
 except
  on E:Exception do BugReport(E,nil,'TextAddMetaData');
 end;
end;

function FileAddMetaDataList(const aFile:LongString; const aMetaText:LongString=''):Integer;
var Lines:TStringList;
begin
 Result:=mde_Missing;
 if IsNonEmptyStr(aFile) then
 if FileExists(aFile) then
 try
  Lines:=TStringList.Create;
  try
   Lines.LoadFromFile(aFile);
   TextAddMetaData(Lines,aMetaText);
   Lines.SaveToFile(aFile);
   Result:=FileCheckMetaDataList(aFile);
  finally
   Kill(Lines);
  end;
 except
  on E:Exception do begin BugReport(E,nil,'FileAddMetaDataList'); Result:=mde_Failure; end;
 end;
end;

function TextCheckMetaData(const aText:LongString):Integer;
var MetaData,TheText,md,cs:LongString;
begin
 Result:=mde_Missing;
 if Length(aText)>0 then
 try
  cs:='';
  MetaData:=TextGetMetaData(aText);
  if Length(MetaData)>0 then begin
   if ScanVarString(svConfig,PChar(MetaData),'@CheckSum%s',cs)<>nil then begin
    cs:=Hex_Decode(cs);
    TheText:=TextDropMetaData(aText);
    md:=GetMD5FromText(UnifyEol(TheText),df_Bin);
    if (Length(cs)>0) and (cs=md)
    then Result:=mde_Success
    else Result:=mde_Invalid;
   end;
  end;
 except
  on E:Exception do begin BugReport(E,nil,'TextCheckMetaData'); Result:=mde_Failure; end;
 end;
end;

function FileCheckMetaDataList(const aFile:LongString; Notify:Boolean=false):Integer;
var Lines:TStringList;
begin
 Result:=mde_Missing;
 if IsNonEmptyStr(aFile) then
 if FileExists(aFile) then
 try
  Lines:=TStringList.Create;
  try
   Lines.LoadFromFile(aFile);
   Result:=TextCheckMetaData(Lines.Text);
   if Notify then SendToMainConsole(Format('@silent @integrity %s:[MetaData]:@CheckSum %s',
                                   [UpCaseStr(mde_Message(Result)),aFile])+EOL)
  finally
   Kill(Lines);
  end;
 except
  on E:Exception do begin BugReport(E,nil,'FileCheckMetaDataList'); Result:=mde_Failure; end;
 end;
end;

function FileCheckMetaData(const aFile:LongString; Notify:Boolean=false):Integer;
var sf:Text; md5:TMD5Digest; sBuff:PChar; Line,Sect,cs,md,s:LongString;
    IsMetaData:Boolean; fsz:Int64; BuffSize:Integer;
const BuffSizeMin=1024; BuffSizeMax=MaxTextBufSize;
begin
 Result:=mde_Missing;
 if IsNonEmptyStr(aFile) then
 if FileExists(aFile) then
 try
  SetInOutRes(0); System.Assign(sf,aFile);
  md5:=nil; sBuff:=nil; IsMetaData:=false; s:=''; cs:=''; md:=''; Sect:='';
  try
   fsz:=GetFileSize64(aFile);
   BuffSize:=EnsureRange(AdjustBufferSize(fsz,BuffSizeMin),BuffSizeMin,BuffSizeMax);
   GetMem(sBuff,BuffSize); FillChar(sBuff^,BuffSize,0); SetTextBuf(sf,sBuff^,BuffSize);
   System.Reset(sf); if System.IOResult<>0 then FatalBug(RusEng('Не могу открыть ','Could not open ')+aFile);
   md5:=NewMD5Digest; md5.Start;
   while not System.Eof(sf) do begin
    System.Readln(sf,Line);
    if System.IoResult<>0 then FatalBug(RusEng('Ошибка чтения ','Error read ')+aFile);
    if ExtractSectionTitle(Line,Sect) then begin
     if SameText(Sect,MetaDataSectionName) then begin
      IsMetaData:=true;
      continue;
     end;
     if IsMetaData and SameText(Sect,EmptySectionName) then begin
      IsMetaData:=false;
      continue;
     end;
     IsMetaData:=false;
    end;
    if IsMetaData then begin
     if ScanVarString(svConfig,PChar(Line),'@CheckSum%s',s)<>nil then cs:=Hex_Decode(s);
    end else begin
     Line:=Line+EOL;
     md5.AddBuffer(UnifyEol(Line));
    end;
   end;
   md5.Stop; md:=md5.Digest[df_Bin];
   if Length(cs)=0 then Result:=mde_Missing else if md=cs then Result:=mde_Success else Result:=mde_Invalid;
   if Notify then SendToMainConsole(Format('@silent @integrity %s:[MetaData]:@CheckSum %s',
                                   [UpCaseStr(mde_Message(Result)),aFile])+EOL);
  finally
   Line:='';
   SmartFileClose(sf); 
   if Assigned(md5) then Kill(md5);
   if Assigned(sBuff) then FreeMem(sBuff);
  end;
 except
  on E:Exception do begin BugReport(E,nil,'FileCheckMetaData'); Result:=mde_Failure; end;
 end;
end;

function FileAddMetaData(const aFile:LongString; const aMetaText:LongString=''):Integer;
var sf,tf:Text; md5:TMD5Digest; sBuff,tBuff:PChar; Line,Sect,tFile,cs,md,dt:LongString;
    IsMetaData:Boolean; fsz:Int64; BuffSize:Integer; ms:Double; Meta:TStringList;
const BuffSizeMin=1024; BuffSizeMax=MaxTextBufSize;
begin
 Result:=mde_Missing;
 if IsNonEmptyStr(aFile) then
 if FileExists(aFile) then
 try
  tFile:=Trim(aFile)+'.tmp';
  SetInOutRes(0); System.Assign(sf,aFile); System.Assign(tf,tFile);
  md5:=nil; sBuff:=nil; tBuff:=nil; IsMetaData:=false; cs:=''; md:=''; Sect:='';
  try
   fsz:=GetFileSize64(aFile);
   BuffSize:=EnsureRange(AdjustBufferSize(fsz,BuffSizeMin),BuffSizeMin,BuffSizeMax);
   GetMem(sBuff,BuffSize); FillChar(sBuff^,BuffSize,0); SetTextBuf(sf,sBuff^,BuffSize);
   GetMem(tBuff,BuffSize); FillChar(tBuff^,BuffSize,0); SetTextBuf(tf,tBuff^,BuffSize);
   System.Reset(sf); if System.IOResult<>0 then FatalBug(RusEng('Не могу открыть ','Could not open ')+aFile);
   System.Rewrite(tf); if System.IOResult<>0 then FatalBug(RusEng('Не могу создать ','Could not create ')+tFile);
   md5:=NewMD5Digest; md5.Start;
   while not System.Eof(sf) do begin
    System.Readln(sf,Line);
    if System.IoResult<>0 then FatalBug(RusEng('Ошибка чтения ','Error read ')+aFile);
    if ExtractSectionTitle(Line,Sect) then begin
     if SameText(Sect,MetaDataSectionName) then begin
      IsMetaData:=true;
      continue;
     end;
     if IsMetaData and SameText(Sect,EmptySectionName) then begin
      IsMetaData:=false;
      continue;
     end;
     IsMetaData:=false;
    end;
    if not IsMetaData then begin
     Line:=Line+EOL;
     md5.AddBuffer(UnifyEol(Line));
     System.Write(tf,Line);
     if System.IOResult<>0 then FatalBug(RusEng('Не могу записать ','Could not write ')+tFile);
    end;
   end;
   md5.Stop; md:=md5.Digest[df_Bin];
   ms:=mSecNow;
   cs:=Hex_Encode(md);
   dt:=GetDateStr(ms,'.',true)+'-'+GetTimeStr(ms,':');
   System.Writeln(tf,MetaDataSectionName);
   System.Writeln(tf,Format('@CheckSum=%s',[cs]));
   System.Writeln(tf,Format('@DateTime=%s',[dt]));
   System.Writeln(tf,Format('@TimeStamp=%g',[ms]));
   if Length(aMetaText)>0 then begin
    Meta:=TStringList.Create;
    try
     Meta.Text:=AdjustLineBreaks(aMetaText);
     System.Write(tf,Meta.Text);
    finally
     Kill(Meta);
    end;
   end;
   System.Writeln(tf,EmptySectionName);
   if System.IOResult<>0 then FatalBug(RusEng('Не могу записать ','Could not write ')+tFile);
  finally
   Line:='';
   SmartFileClose(sf);
   SmartFileClose(tf);
   if Assigned(md5) then Kill(md5);
   if Assigned(sBuff) then FreeMem(sBuff);
   if Assigned(tBuff) then FreeMem(tBuff);
  end;
  if not FileRename(tFile,aFile) then FileErase(tFile);
  Result:=FileCheckMetaData(aFile);
 except
  on E:Exception do begin BugReport(E,nil,'FileAddMetaData'); Result:=mde_Failure; end;
 end;
end;

function mde_Message(mde:Integer):LongString;
begin
 case mde of
  mde_Success : Result:='Success';
  mde_Missing : Result:='Missing';
  mde_Invalid : Result:='Invalid';
  else          Result:='Failure';
 end;
end;

function TextMetaDataUtility(Arg1,Arg2,Arg3:LongString):Integer;
var SkipEcho:Boolean;
 function Help:Integer;
 begin
  SkipEcho:=True;
  Result:=mde_Success;
  Echo('Help on TextMetaData utility:');
  Echo('Copyright (c) 2017-2026 Alexey Kuryakin daqgroup@mail.ru');
  Echo('TextMetaData uses to add [MetaData] section and @CheckSum to the end of text');
  Echo('file for data integrity verification.  MetaData section should be ignored by');
  Echo('text file readers, and MetaData may be modified without changes of checksum.');
  Echo('TextMetaData tool may be used to save/verify checksum of any text files with');
  Echo('[..] section structure like INI files or configuration files.');
  Echo('Usage:');
  Echo('TextMetaData -a File [Meta] - [MetaData] @CheckSum and Meta text add to File');
  Echo('                              Meta text (if one exist) should be URL-encoded');
  Echo('TextMetaData -c File        - verify data integrity of text contents of File');
  Echo('TextMetaData -h             - show this help screen');
  Echo('Also -a option have synonym --add, -c synonym --check, -h synonym --help');
  Echo('Return:  0=Success,  1=Missing file/section,  2=Invalid checksum,  3=Failure');
 end;
begin
 Result:=mde_Missing;
 try
  SkipEcho:=false;
  try
   if IsEmptyStr(Arg1) or SameText(Arg1,'-h') or SameText(Arg1,'--help') then begin
    Result:=Help;
   end else
   if SameText(Arg1,'-a') or SameText(Arg1,'--add') then begin
    Result:=FileAddMetaData(Arg2,URL_Decode(Arg3));
   end else
   if SameText(Arg1,'-c') or SameText(Arg1,'--check') then begin
    Result:=FileCheckMetaData(Arg2);
   end else
   Exit;
  finally
   if not SkipEcho then
   Echo(Format('%s [MetaData] @CheckSum %s',[UpCaseStr(mde_Message(Result)),Trim(Arg2)]));
  end;
 except
  on E:Exception do begin BugReport(E,nil,'TextMetaDataUtility'); Result:=mde_Failure; end;
 end;
end;

///////////////////////////////////////
// Unit initialization and finalization
///////////////////////////////////////

procedure Init_crw_meta;
begin
end;

procedure Free_crw_meta;
begin
end;

initialization

 Init_crw_meta;

finalization

 Free_crw_meta;

end.

//////////////
// END OF FILE
//////////////

