Updated components, added 2 new units
This commit is contained in:
Christian Hammacher
608
editor/studio/UtfFunct.PAS
Executable file
608
editor/studio/UtfFunct.PAS
Executable file
@ -0,0 +1,608 @@
|
||||
{
|
||||
This file is translated to Delphi from the file referenced below
|
||||
by Jan Martin Pettersen (hdalis@users.sourceforge.net)
|
||||
23/07/2005.
|
||||
|
||||
Some code in this file is also taken from the SciTE (Neil Hodgson)
|
||||
}
|
||||
// Utf8_16.cxx
|
||||
// Copyright (C) 2002 Scott Kirkwood
|
||||
//
|
||||
// Permission to use, copy, modify, distribute and sell this code
|
||||
// and its documentation for any purpose is hereby granted without fee,
|
||||
// provided that the above copyright notice appear in all copies or
|
||||
// any derived copies. Scott Kirkwood makes no representations
|
||||
// about the suitability of this software for any purpose.
|
||||
// It is provided "as is" without express or implied warranty.
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
unit UtfFunct;
|
||||
interface
|
||||
|
||||
uses Windows,SysUtils,Classes,Math;
|
||||
const
|
||||
UniBufSize=32000;
|
||||
|
||||
type
|
||||
Utf16=Word;
|
||||
Utf8=Byte;
|
||||
TUtf8Array=array[0..1] of Utf8;
|
||||
PUtf8=^TUtf8Array;
|
||||
TUtf16Array=array[0..1] of Utf16;
|
||||
PUtf16=^TUtf16Array;
|
||||
|
||||
//uniCookie isn't used yet..
|
||||
UniMode=(uni8Bit, uni16BE, uni16LE, uniUTF8,uniCookie);
|
||||
//States for the unicode Next functions..
|
||||
eState=(eStart,e2Bytes2,e3Bytes2,e3Bytes3);
|
||||
|
||||
// Reads UTF-16 and outputs UTF-8
|
||||
Utf16_Iter=class(TObject)
|
||||
private
|
||||
m_eEncoding : UniMode;
|
||||
m_eState : eState;
|
||||
m_pBuf : PByte;
|
||||
m_pRead : PByte;
|
||||
m_pEnd : PByte;
|
||||
m_nCur : Utf8;
|
||||
m_nCur16 : Utf16;
|
||||
public
|
||||
constructor Create;
|
||||
procedure Reset;
|
||||
procedure Set_(const pbuf : PByte;nLen : Cardinal;eEncoding : UniMode);
|
||||
function More : Boolean;
|
||||
procedure Next;
|
||||
function Get : Utf8;
|
||||
end;
|
||||
|
||||
|
||||
// Reads UTF-8 and outputs UTF-16
|
||||
Utf8_Iter =class(TObject)
|
||||
private
|
||||
m_eEncoding : UniMode;
|
||||
m_eState : eState;
|
||||
m_pBuf : PByte;
|
||||
m_pRead : PByte;
|
||||
m_pEnd : PByte;
|
||||
m_nCur16 : Utf16;
|
||||
procedure toStart;
|
||||
procedure Swap;
|
||||
public
|
||||
constructor Create;
|
||||
procedure Reset;
|
||||
procedure Set_(const pbuf : PByte;nLen : Cardinal;eEncoding : UniMode);
|
||||
function More : Boolean; //bool
|
||||
procedure Next;
|
||||
function Get : Utf16;
|
||||
function canGet : Boolean;
|
||||
end;
|
||||
|
||||
// Reads UTF16 and outputs UTF8
|
||||
UtfRead=class(TObject)
|
||||
private
|
||||
m_eEncoding : UniMode;
|
||||
m_pBuf : PByte;
|
||||
m_nBufSize : Cardinal;
|
||||
m_bFirstRead : Boolean;
|
||||
m_pNewBuf : PByte;
|
||||
m_nLen : Cardinal;
|
||||
m_Iter16 : Utf16_Iter;
|
||||
public
|
||||
constructor Create;
|
||||
destructor Destroy;override;
|
||||
function getEncoding : UniMode;
|
||||
function getNewBuf : PChar;
|
||||
function Convert(buf : PChar; len : Cardinal) : Cardinal;
|
||||
procedure Reset;
|
||||
property Encoding : UniMode read m_eEncoding;
|
||||
end;
|
||||
|
||||
// Read in a UTF-8 buffer and write out to UTF-16 or UTF-8
|
||||
UtfWrite=class(TObject)
|
||||
private
|
||||
m_eEncoding : UniMode;
|
||||
m_pBuf : PUtf16;
|
||||
m_nBufSize : Cardinal;
|
||||
m_bFirstWrite : Boolean;
|
||||
m_pFile : TStream;
|
||||
procedure SetDestStream(Value : TStream);
|
||||
procedure SetEncoding(eType : UniMode);
|
||||
public
|
||||
constructor Create;
|
||||
function Write(const Buffer; Count : Cardinal) : LongInt;
|
||||
property DestStream : TStream read m_pFile write SetDestStream;
|
||||
property Encoding : UniMode read m_eEncoding write SetEncoding;
|
||||
end;
|
||||
|
||||
//Returns the UTF8 length of the buffer 'uptr'.
|
||||
function UTF8Length(const wideSrc : PWideChar; wideLen : Cardinal) : Cardinal;
|
||||
//Transforms UCS2 to UTF8.
|
||||
procedure UTF8FromUCS2(const wideSrc : PWideChar; wideLen : Cardinal; utfDestBuf : PChar; utfDestLen : Cardinal);
|
||||
function UTF8ToAnsiP(const srcBuffer : PChar;len : Integer;destBuffer : PChar) : Integer;
|
||||
function DetectEncoding(buf : PByte;len : Integer;var Encoding : UniMode) : Integer;
|
||||
|
||||
implementation
|
||||
|
||||
const
|
||||
k_boms : array[uni8bit..uniUTF8,0..2] of Utf8=(
|
||||
($00,$00,$00),
|
||||
($FE,$FF,$00),
|
||||
($FF,$FE,$00),
|
||||
($EF,$BB,$BF));
|
||||
|
||||
function UTF8ToAnsiP(const srcBuffer : PChar;len : Integer;destBuffer : PChar) : Integer;
|
||||
var
|
||||
tmpbuffer : String;
|
||||
srcLen,i,destLen : Integer;
|
||||
begin
|
||||
Result:=0;
|
||||
if (not assigned(srcBuffer)) or (not assigned(destBuffer)) then Exit;
|
||||
if len=-1 then
|
||||
srcLen:=Length(srcBuffer)
|
||||
else
|
||||
srcLen:=len;
|
||||
tmpbuffer:=UTF8ToAnsi(Copy(srcBuffer,1,srcLen));
|
||||
destLen:=Length(tmpbuffer);
|
||||
for i:=1 to destLen do
|
||||
destBuffer[i-1]:=tmpbuffer[i];
|
||||
destBuffer[destLen]:=#0;
|
||||
Result:=destLen;
|
||||
end;
|
||||
|
||||
function UTF8Length(const wideSrc : PWideChar; wideLen : Cardinal) : Cardinal;
|
||||
var
|
||||
i,len : Cardinal;
|
||||
uch : Cardinal;
|
||||
begin
|
||||
len := 0;
|
||||
i:=0;
|
||||
while((i<wideLen) and (Cardinal(wideSrc[i])<>0)) do
|
||||
begin
|
||||
uch:=Cardinal(wideSrc[i]);
|
||||
if (uch < $80) then
|
||||
Inc(len)
|
||||
else if (uch < $800) then
|
||||
Inc(len,2)
|
||||
else
|
||||
Inc(len,3);
|
||||
Inc(i);
|
||||
end;
|
||||
Result:=len;
|
||||
end;
|
||||
|
||||
procedure UTF8FromUCS2(const wideSrc : PWideChar; wideLen : Cardinal; utfDestBuf : PChar; utfDestLen : Cardinal);
|
||||
var
|
||||
k : Integer;
|
||||
i : Cardinal;
|
||||
uch : Cardinal;
|
||||
begin
|
||||
k:= 0;
|
||||
i:=0;
|
||||
while((i<wideLen)and(Cardinal(wideSrc[i])<>0)) do
|
||||
begin
|
||||
uch:=Cardinal(wideSrc[i]);
|
||||
if uch<$80 then
|
||||
begin
|
||||
utfDestBuf[k] := Char(uch);
|
||||
Inc(k);
|
||||
end else
|
||||
if (uch<$800) then
|
||||
begin
|
||||
utfDestBuf[k]:=Char($C0 or (uch shr 6));
|
||||
Inc(k);
|
||||
utfDestBuf[k] := Char($80 or (uch and $3f));
|
||||
Inc(k);
|
||||
end else
|
||||
begin
|
||||
utfDestBuf[k] := Char($E0 or (uch shr 12));
|
||||
Inc(k);
|
||||
utfDestBuf[k] := Char($80 or ((uch shr 6) and $3f));
|
||||
Inc(k);
|
||||
utfDestBuf[k] := Char($80 or (uch and $3f));
|
||||
Inc(k);
|
||||
end;
|
||||
end;
|
||||
utfDestBuf[utfDestLen]:=#0;
|
||||
end;
|
||||
|
||||
function DetectEncoding(buf : PByte;len : Integer;var Encoding : UniMode) : Integer;
|
||||
var
|
||||
nRet : Integer;
|
||||
pbTmp : PByteArray;
|
||||
begin
|
||||
Encoding := uni8bit;
|
||||
pbTmp:=PByteArray(buf);
|
||||
nRet := 0;
|
||||
if (len > 1) then
|
||||
begin
|
||||
if ((pbTmp[0]=k_Boms[uni16BE][0]) and (pbTmp[1]=k_Boms[uni16BE][1])) then
|
||||
begin
|
||||
Encoding := uni16BE;
|
||||
nRet := 2;
|
||||
end else
|
||||
if ((pbTmp[0]=k_Boms[uni16LE][0]) and (pbTmp[1]=k_Boms[uni16LE][1])) then
|
||||
begin
|
||||
Encoding := uni16LE;
|
||||
nRet := 2;
|
||||
end else
|
||||
if ((len>2) and (pbTmp[0]=k_Boms[uniUTF8][0]) and (pbTmp[1]=k_Boms[uniUTF8][1]) and (pbTmp[2]=k_Boms[uniUTF8][2])) then
|
||||
begin
|
||||
Encoding := uniUTF8;
|
||||
nRet := 3;
|
||||
end;
|
||||
end;
|
||||
Result:=nRet;
|
||||
end;
|
||||
|
||||
|
||||
constructor Utf16_Iter.Create;
|
||||
begin
|
||||
Reset;
|
||||
end;
|
||||
procedure Utf16_Iter.Reset;
|
||||
begin
|
||||
m_pBuf := nil;
|
||||
m_pRead := nil;
|
||||
m_pEnd := nil;
|
||||
m_eState := eStart;
|
||||
m_nCur := 0;
|
||||
m_nCur16 := 0;
|
||||
m_eEncoding := uni8bit;
|
||||
end;
|
||||
procedure Utf16_Iter.Set_(const pbuf : PByte;nLen : Cardinal;eEncoding : UniMode);
|
||||
begin
|
||||
m_pBuf := pBuf;
|
||||
m_pRead := pBuf;
|
||||
m_pEnd := pBuf;
|
||||
Inc(m_pEnd,nLen);
|
||||
m_eEncoding := eEncoding;
|
||||
Next;
|
||||
end;
|
||||
|
||||
procedure Utf16_Iter.Next;
|
||||
begin
|
||||
case m_eState of
|
||||
eStart:
|
||||
begin
|
||||
if (m_eEncoding = uni16LE) then
|
||||
begin
|
||||
m_nCur16 := Utf16(m_pRead^);
|
||||
Inc(m_pRead);
|
||||
m_nCur16 := m_nCur16 or Utf16((m_pRead^ shl 8));
|
||||
end else
|
||||
begin
|
||||
m_nCur16 := Utf16(m_pRead^ shl 8);
|
||||
Inc(m_pRead);
|
||||
m_nCur16 := m_nCur16 or m_pRead^;
|
||||
end;
|
||||
Inc(m_pRead);
|
||||
|
||||
if (m_nCur16 < $80) then
|
||||
begin
|
||||
m_nCur := Byte(m_nCur16 and $FF);
|
||||
m_eState := eStart;
|
||||
end else
|
||||
if (m_nCur16 < $800) then
|
||||
begin
|
||||
m_nCur := Byte($C0 or (m_nCur16 shr 6));
|
||||
m_eState := e2Bytes2;
|
||||
end else
|
||||
begin
|
||||
m_nCur := Byte($E0 or (m_nCur16 shr 12));
|
||||
m_eState := e3Bytes2;
|
||||
end;
|
||||
|
||||
end;
|
||||
e2Bytes2:
|
||||
begin
|
||||
m_nCur := Byte($80 or (m_nCur16 and $3F));
|
||||
m_eState := eStart;
|
||||
end;
|
||||
e3Bytes2:
|
||||
begin
|
||||
m_nCur := Byte($80 or ((m_nCur16 shr 6) and $3F));
|
||||
m_eState := e3Bytes3;
|
||||
end;
|
||||
e3Bytes3:
|
||||
begin
|
||||
m_nCur := Byte($80 or (m_nCur16 and $3F));
|
||||
m_eState := eStart;
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
function Utf16_Iter.More : Boolean;
|
||||
begin
|
||||
Result:=Cardinal(m_pRead) <= Cardinal(m_pEnd);
|
||||
end;
|
||||
|
||||
function Utf16_Iter.Get : Utf8;
|
||||
begin
|
||||
Result:=m_nCur;
|
||||
end;
|
||||
|
||||
constructor Utf8_Iter.Create;
|
||||
begin
|
||||
Reset;
|
||||
end;
|
||||
|
||||
procedure Utf8_Iter.Reset;
|
||||
begin
|
||||
m_pBuf := nil;
|
||||
m_pRead := nil;
|
||||
m_pEnd := nil;
|
||||
m_eState := eStart;
|
||||
m_nCur16 := 0;
|
||||
m_eEncoding := uni8bit;
|
||||
end;
|
||||
|
||||
procedure Utf8_Iter.Set_(const pbuf : PByte;nLen : Cardinal;eEncoding : UniMode);
|
||||
begin
|
||||
m_pBuf := pBuf;
|
||||
m_pRead := pBuf;
|
||||
m_pEnd := pBuf;
|
||||
Inc(m_pEnd,nLen);
|
||||
m_eEncoding := eEncoding;
|
||||
Next;
|
||||
end;
|
||||
|
||||
procedure Utf8_Iter.Next;
|
||||
begin
|
||||
case (m_eState) of
|
||||
eStart:
|
||||
begin
|
||||
if (($E0 and m_pRead^) = $E0) then
|
||||
begin
|
||||
m_nCur16 := Utf16(((not $E0) and m_pRead^) shl 12);
|
||||
m_eState := e3Bytes2;
|
||||
end else if (($C0 and m_pRead^) = $C0) then
|
||||
begin
|
||||
m_nCur16 := Utf16((not $C0 and m_pRead^) shl 6);
|
||||
m_eState := e2Bytes2;
|
||||
end else
|
||||
begin
|
||||
m_nCur16 := m_pRead^;
|
||||
toStart;
|
||||
end;
|
||||
end;
|
||||
e2Bytes2:
|
||||
begin
|
||||
m_nCur16 :=m_nCur16 or utf8($3F and m_pRead^);
|
||||
toStart;
|
||||
end;
|
||||
e3Bytes2:
|
||||
begin
|
||||
m_nCur16 :=m_nCur16 or utf16(($3F and m_pRead^) shl 6);
|
||||
m_eState := e3Bytes3;
|
||||
end;
|
||||
e3Bytes3:
|
||||
begin
|
||||
m_nCur16 :=m_nCur16 or utf8($3F and m_pRead^);
|
||||
toStart;
|
||||
end;
|
||||
end;
|
||||
Inc(m_pRead);
|
||||
end;
|
||||
|
||||
function Utf8_Iter.More : Boolean;
|
||||
begin
|
||||
Result:=Cardinal(m_pRead) <= Cardinal(m_pEnd);
|
||||
end;
|
||||
|
||||
function Utf8_Iter.Get : Utf16;
|
||||
begin
|
||||
Result:=m_nCur16;
|
||||
end;
|
||||
function Utf8_Iter.canGet : Boolean;
|
||||
begin
|
||||
Result:=m_eState = eStart;
|
||||
end;
|
||||
procedure Utf8_Iter.toStart;
|
||||
begin
|
||||
m_eState := eStart;
|
||||
if (m_eEncoding = uni16BE) then
|
||||
Swap;
|
||||
end;
|
||||
procedure Utf8_Iter.Swap;
|
||||
var
|
||||
p : PUtf8;
|
||||
swapbyte : Utf8;
|
||||
begin
|
||||
p := PUtf8(@m_nCur16);
|
||||
swapbyte := p[0];
|
||||
p[0]:= p[1];
|
||||
p[1]:=swapbyte;
|
||||
end;
|
||||
|
||||
constructor UtfRead.Create;
|
||||
begin
|
||||
m_eEncoding := uni8bit;
|
||||
m_nBufSize := 0;
|
||||
m_pNewBuf := nil;
|
||||
m_bFirstRead := True;
|
||||
end;
|
||||
|
||||
destructor UtfRead.Destroy;
|
||||
begin
|
||||
if ((m_eEncoding <> uni8bit) and (m_eEncoding <> uniUTF8)) then
|
||||
begin
|
||||
if assigned(m_pNewBuf) then FreeMem(m_pNewBuf);
|
||||
end;
|
||||
inherited;
|
||||
end;
|
||||
|
||||
function UtfRead.getEncoding : UniMode;
|
||||
begin
|
||||
Result:=m_eEncoding;
|
||||
end;
|
||||
function UtfRead.getNewBuf : PChar;
|
||||
begin
|
||||
Result:=PChar(m_pNewBuf);
|
||||
end;
|
||||
|
||||
procedure UtfRead.Reset;
|
||||
begin
|
||||
m_bFirstRead:=True;
|
||||
m_nBufSize:=0;
|
||||
m_eEncoding :=uni8Bit;
|
||||
end;
|
||||
|
||||
function UtfRead.Convert(buf : PChar; len : Cardinal) : Cardinal;
|
||||
var
|
||||
nSkip : Cardinal;
|
||||
newSize : Cardinal;
|
||||
pCur,pTemp : PByte;
|
||||
begin
|
||||
m_Iter16:=Utf16_Iter.Create;
|
||||
try
|
||||
m_pBuf := PByte(buf);
|
||||
m_nLen := len;
|
||||
|
||||
nSkip := 0;
|
||||
if (m_bFirstRead) then
|
||||
begin
|
||||
nSkip := DetectEncoding(m_pBuf,m_nLen,m_eEncoding);
|
||||
m_bFirstRead := False;
|
||||
end;
|
||||
|
||||
if (m_eEncoding = uni8bit) then
|
||||
begin
|
||||
// Do nothing, pass through
|
||||
m_nBufSize := 0;
|
||||
m_pNewBuf := m_pBuf;
|
||||
Result:=len;
|
||||
Exit;
|
||||
end;
|
||||
|
||||
if (m_eEncoding = uniUTF8) then
|
||||
begin
|
||||
// Pass through after BOM
|
||||
m_nBufSize := 0;
|
||||
m_pNewBuf := m_pBuf;
|
||||
Inc(m_pNewBuf,nSkip);
|
||||
Result:=len - nSkip;
|
||||
Exit;
|
||||
end;
|
||||
|
||||
// Else...
|
||||
//newSize := len + len div 2 + 1;
|
||||
newSize:=len*2+1;
|
||||
if (m_nBufSize <> newSize) then
|
||||
begin
|
||||
FreeMem(m_pNewBuf);
|
||||
m_pNewBuf:=nil;
|
||||
GetMem(m_pNewBuf,newSize);
|
||||
m_nBufSize := newSize;
|
||||
end;
|
||||
|
||||
pCur := m_pNewBuf;
|
||||
pTemp:=m_pBuf;
|
||||
Inc(pTemp,nSkip);
|
||||
m_Iter16.Set_(pTemp, len - nSkip, m_eEncoding);
|
||||
while(m_Iter16.More) do
|
||||
begin
|
||||
pCur^:=m_Iter16.Get;
|
||||
Inc(PCur);
|
||||
m_Iter16.Next;
|
||||
end;
|
||||
// Return number of bytes writen out
|
||||
finally
|
||||
FreeAndNil(m_Iter16);
|
||||
end;
|
||||
Result:=Cardinal(pCur) - Cardinal(m_pNewBuf);
|
||||
end;
|
||||
|
||||
constructor UtfWrite.Create;
|
||||
begin
|
||||
m_eEncoding := uni8bit;
|
||||
m_pFile := nil;
|
||||
m_pBuf := nil;
|
||||
m_bFirstWrite := true;
|
||||
m_nBufSize := 0;
|
||||
end;
|
||||
|
||||
procedure UtfWrite.SetEncoding(eType : UniMode);
|
||||
begin
|
||||
m_eEncoding := eType;
|
||||
end;
|
||||
procedure UtfWrite.SetDestStream(Value : TStream);
|
||||
begin
|
||||
m_pFile:=Value;
|
||||
m_bFirstWrite:=True;
|
||||
end;
|
||||
|
||||
function UtfWrite.Write(const Buffer; Count : Cardinal) : LongInt;
|
||||
var
|
||||
iter8 : Utf8_Iter;
|
||||
pCur : ^Utf16;
|
||||
ret : LongInt;
|
||||
pTemp : PChar;
|
||||
begin
|
||||
if Count=0 then
|
||||
begin
|
||||
Result:=0;
|
||||
Exit;
|
||||
end;
|
||||
iter8:=Utf8_Iter.Create;
|
||||
try
|
||||
if (not assigned(m_pFile)) then
|
||||
begin
|
||||
Result:=0;
|
||||
Exit;
|
||||
end;
|
||||
if (m_eEncoding = uni8bit) then
|
||||
begin
|
||||
// Normal write
|
||||
m_bFirstWrite:=False;
|
||||
Result:=m_pFile.Write(PChar(Buffer)^, Count);
|
||||
Exit;
|
||||
end;
|
||||
|
||||
if (m_eEncoding = uniUTF8) then
|
||||
begin
|
||||
pTemp:=PChar(Buffer);
|
||||
if (m_bFirstWrite) then
|
||||
begin
|
||||
m_pFile.Write(k_Boms[m_eEncoding], 3);
|
||||
m_bFirstWrite := false;
|
||||
end;
|
||||
Result:=m_pFile.Write(pTemp^, Count);
|
||||
Exit;
|
||||
end;
|
||||
|
||||
if (Count > m_nBufSize) then
|
||||
begin
|
||||
m_nBufSize := Count;
|
||||
if assigned(m_pBuf) then FreeMem(m_pBuf);
|
||||
m_pBuf := nil;
|
||||
GetMem(m_pBuf,SizeOf(Utf16)*(Count+1));
|
||||
end;
|
||||
|
||||
if (m_bFirstWrite) then
|
||||
begin
|
||||
if ((m_eEncoding = uni16BE) or (m_eEncoding = uni16LE)) then
|
||||
begin
|
||||
// Write the BOM
|
||||
m_pFile.Write(k_Boms[m_eEncoding],2);
|
||||
end;
|
||||
m_bFirstWrite := false;
|
||||
end;
|
||||
|
||||
iter8.set_(PByte(Buffer), Count, m_eEncoding);
|
||||
pCur := @m_pBuf[0];
|
||||
while(iter8.More) do
|
||||
begin
|
||||
if (iter8.canGet) then
|
||||
begin
|
||||
pCur^ := iter8.Get;
|
||||
Inc(pCur);
|
||||
end;
|
||||
iter8.Next;
|
||||
end;
|
||||
ret := m_pFile.Write(m_pBuf^,Cardinal(pCur)-Cardinal(m_pBuf));
|
||||
finally
|
||||
if assigned(iter8) then FreeAndNil(iter8);
|
||||
end;
|
||||
Result:=ret;
|
||||
end;
|
||||
|
||||
end.
|
||||
Reference in New Issue
Block a user