ag/cpython-source-deps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import Tcl 8.6.11

Browse Source
This commit is contained in:
Steve Dower
2021-03-30 00:51:39 +01:00
parent 3bb8e3e086
commit 1aadb2455c
923 changed files with 79104 additions and 62616 deletions
Download Patch File Download Diff File Expand all files Collapse all files

432
generic/tclEncoding.c
View File

@@ -18,7 +18,7 @@ typedef size_t (LengthProc)(const char *src);
* convert between various character sets and UTF-8.
*/
typedef struct Encoding {
typedef struct {
char *name; /* Name of encoding. Malloced because (1) hash
* table entry that owns this encoding may be
* freed prior to this encoding being freed,
@@ -57,7 +57,7 @@ typedef struct Encoding {
* encoding.
*/
typedef struct TableEncodingData {
typedef struct {
int fallback; /* Character (in this encoding) to substitute
* when this encoding cannot represent a UTF-8
* character. */
@@ -83,7 +83,7 @@ typedef struct TableEncodingData {
} TableEncodingData;
/*
* The following structures is the clientData for a dynamically-loaded,
* Each of the following structures is the clientData for a dynamically-loaded
* escape-driven encoding that is itself comprised of other simpler encodings.
* An example is "iso-2022-jp", which uses escape sequences to switch between
* ascii, jis0208, jis0212, gb2312, and ksc5601. Note that "escape-driven"
@@ -91,7 +91,7 @@ typedef struct TableEncodingData {
* for switching character sets.
*/
typedef struct EscapeSubTable {
typedef struct {
unsigned sequenceLen; /* Length of following string. */
char sequence[16]; /* Escape code that marks this encoding. */
char name[32]; /* Name for encoding. */
@@ -100,7 +100,7 @@ typedef struct EscapeSubTable {
* yet. */
} EscapeSubTable;
typedef struct EscapeEncodingData {
typedef struct {
int fallback; /* Character (in this encoding) to substitute
* when this encoding cannot represent a UTF-8
* character. */
@@ -116,9 +116,9 @@ typedef struct EscapeEncodingData {
* entry in this array is 1, otherwise it is
* 0. */
int numSubTables; /* Length of following array. */
EscapeSubTable subTables[1];/* Information about each EscapeSubTable used
* by this encoding type. The actual size will
* be as large as necessary to hold all
EscapeSubTable subTables[TCLFLEXARRAY];/* Information about each EscapeSubTable used
* by this encoding type. The actual size is
* as large as necessary to hold all
* EscapeSubTables. */
} EscapeEncodingData;
@@ -156,7 +156,7 @@ static ProcessGlobalValue encodingFileMap = {
* A list of directories making up the "library path". Historically this
* search path has served many uses, but the only one remaining is a base for
* the encodingSearchPath above. If the application does not explicitly set
* the encodingSearchPath, then it will be initialized by appending /encoding
* the encodingSearchPath, then it is initialized by appending /encoding
* to each directory in this "libraryPath".
*/
@@ -177,7 +177,7 @@ TCL_DECLARE_MUTEX(encodingMutex)
/*
* The following are used to hold the default and current system encodings.
* If NULL is passed to one of the conversion routines, the current setting of
* the system encoding will be used to perform the conversion.
* the system encoding is used to perform the conversion.
*/
static Tcl_Encoding defaultEncoding = NULL;
@@ -195,35 +195,25 @@ static unsigned short emptyPage[256];
* Functions used only in this module.
*/
static int BinaryProc(ClientData clientData,
const char *src, int srcLen, int flags,
Tcl_EncodingState *statePtr, char *dst, int dstLen,
int *srcReadPtr, int *dstWrotePtr,
int *dstCharsPtr);
static void DupEncodingIntRep(Tcl_Obj *srcPtr, Tcl_Obj *dupPtr);
static void EscapeFreeProc(ClientData clientData);
static int EscapeFromUtfProc(ClientData clientData,
const char *src, int srcLen, int flags,
Tcl_EncodingState *statePtr, char *dst, int dstLen,
int *srcReadPtr, int *dstWrotePtr,
int *dstCharsPtr);
static int EscapeToUtfProc(ClientData clientData,
const char *src, int srcLen, int flags,
Tcl_EncodingState *statePtr, char *dst, int dstLen,
int *srcReadPtr, int *dstWrotePtr,
int *dstCharsPtr);
static void FillEncodingFileMap(void);
static void FreeEncoding(Tcl_Encoding encoding);
static void FreeEncodingIntRep(Tcl_Obj *objPtr);
static Encoding * GetTableEncoding(EscapeEncodingData *dataPtr,
int state);
static Tcl_Encoding LoadEncodingFile(Tcl_Interp *interp, const char *name);
static Tcl_Encoding LoadTableEncoding(const char *name, int type,
Tcl_Channel chan);
static Tcl_Encoding LoadEscapeEncoding(const char *name, Tcl_Channel chan);
static Tcl_Channel OpenEncodingFileChannel(Tcl_Interp *interp,
const char *name);
static void TableFreeProc(ClientData clientData);
static Tcl_EncodingConvertProc BinaryProc;
static Tcl_DupInternalRepProc DupEncodingIntRep;
static Tcl_EncodingFreeProc EscapeFreeProc;
static Tcl_EncodingConvertProc EscapeFromUtfProc;
static Tcl_EncodingConvertProc EscapeToUtfProc;
static void FillEncodingFileMap(void);
static void FreeEncoding(Tcl_Encoding encoding);
static Tcl_FreeInternalRepProc FreeEncodingIntRep;
static Encoding * GetTableEncoding(EscapeEncodingData *dataPtr,
int state);
static Tcl_Encoding LoadEncodingFile(Tcl_Interp *interp,
const char *name);
static Tcl_Encoding LoadTableEncoding(const char *name, int type,
Tcl_Channel chan);
static Tcl_Encoding LoadEscapeEncoding(const char *name,
Tcl_Channel chan);
static Tcl_Channel OpenEncodingFileChannel(Tcl_Interp *interp,
const char *name);
static Tcl_EncodingFreeProc TableFreeProc;
static int TableFromUtfProc(ClientData clientData,
const char *src, int srcLen, int flags,
Tcl_EncodingState *statePtr, char *dst, int dstLen,
@@ -335,7 +325,7 @@ static void
FreeEncodingIntRep(
Tcl_Obj *objPtr)
{
Tcl_FreeEncoding(objPtr->internalRep.twoPtrValue.ptr1);
Tcl_FreeEncoding((Tcl_Encoding)objPtr->internalRep.twoPtrValue.ptr1);
objPtr->typePtr = NULL;
}
@@ -429,9 +419,8 @@ TclGetLibraryPath(void)
* Keeps the per-thread copy of the library path current with changes to
* the global copy.
*
* NOTE: this routine returns void, so there's no way to report the error
* that searchPath is not a valid list. In that case, this routine will
* silently do nothing.
* Since the result of this routine is void, if searchPath is not a valid
* list this routine silently does nothing.
*
*----------------------------------------------------------------------
*/
@@ -453,17 +442,16 @@ TclSetLibraryPath(
*
* FillEncodingFileMap --
*
* Called to bring the encoding file map in sync with the current value
* Called to update the encoding file map with the current value
* of the encoding search path.
*
* Scan the directories on the encoding search path, find the *.enc
* files, and store the found pathnames in a map associated with the
* encoding name.
* Finds *.end files in the directories on the encoding search path and
* stores the found pathnames in a map associated with the encoding name.
*
* In particular, if $dir is on the encoding search path, and the file
* $dir/foo.enc is found, then store a "foo" -> $dir entry in the map.
* Later, any need for the "foo" encoding will quickly * be able to
* construct the $dir/foo.enc pathname for reading the encoding data.
* If $dir is on the encoding search path and the file $dir/foo.enc is
* found, stores a "foo" -> $dir entry in the map. if the "foo" encoding
* is needed later, the $dir/foo.enc name can be quickly constructed in
* order to read the encoding data.
*
* Results:
* None.
@@ -547,19 +535,24 @@ TclInitEncodingSubsystem(void)
TableEncodingData *dataPtr;
unsigned size;
unsigned short i;
union {
char c;
short s;
} isLe;
if (encodingsInitialized) {
return;
}
isLe.s = 1;
Tcl_MutexLock(&encodingMutex);
Tcl_InitHashTable(&encodingTable, TCL_STRING_KEYS);
Tcl_MutexUnlock(&encodingMutex);
/*
* Create a few initial encodings. Note that the UTF-8 to UTF-8
* translation is not a no-op, because it will turn a stream of improperly
* formed UTF-8 into a properly formed stream.
* Create a few initial encodings. UTF-8 to UTF-8 translation is not a
* no-op because it turns a stream of improperly formed UTF-8 into a
* properly formed stream.
*/
type.encodingName = "identity";
@@ -583,7 +576,7 @@ TclInitEncodingSubsystem(void)
type.fromUtfProc = UtfToUnicodeProc;
type.freeProc = NULL;
type.nullSize = 2;
type.clientData = NULL;
type.clientData = INT2PTR(isLe.c);
Tcl_CreateEncoding(&type);
/*
@@ -593,14 +586,14 @@ TclInitEncodingSubsystem(void)
* code to duplicate the structure of a table encoding here.
*/
dataPtr = ckalloc(sizeof(TableEncodingData));
dataPtr = (TableEncodingData *)ckalloc(sizeof(TableEncodingData));
memset(dataPtr, 0, sizeof(TableEncodingData));
dataPtr->fallback = '?';
size = 256*(sizeof(unsigned short *) + sizeof(unsigned short));
dataPtr->toUnicode = ckalloc(size);
dataPtr->toUnicode = (unsigned short **)ckalloc(size);
memset(dataPtr->toUnicode, 0, size);
dataPtr->fromUnicode = ckalloc(size);
dataPtr->fromUnicode = (unsigned short **)ckalloc(size);
memset(dataPtr->fromUnicode, 0, size);
dataPtr->toUnicode[0] = (unsigned short *) (dataPtr->toUnicode + 256);
@@ -666,7 +659,7 @@ TclFinalizeEncodingSubsystem(void)
* cleaned up.
*/
FreeEncoding(Tcl_GetHashValue(hPtr));
FreeEncoding((Tcl_Encoding)Tcl_GetHashValue(hPtr));
hPtr = Tcl_FirstHashEntry(&encodingTable, &search);
}
@@ -752,11 +745,7 @@ Tcl_SetDefaultEncodingDir(
* interp was NULL.
*
* Side effects:
* The new encoding type is entered into a table visible to all
* interpreters, keyed off the encoding's name. For each call to this
* function, there should eventually be a call to Tcl_FreeEncoding, so
* that the database can be cleaned up when encodings aren't needed
* anymore.
* LoadEncodingFile is called if necessary.
*
*-------------------------------------------------------------------------
*/
@@ -779,7 +768,7 @@ Tcl_GetEncoding(
hPtr = Tcl_FindHashEntry(&encodingTable, name);
if (hPtr != NULL) {
encodingPtr = Tcl_GetHashValue(hPtr);
encodingPtr = (Encoding *)Tcl_GetHashValue(hPtr);
encodingPtr->refCount++;
Tcl_MutexUnlock(&encodingMutex);
return (Tcl_Encoding) encodingPtr;
@@ -794,15 +783,15 @@ Tcl_GetEncoding(
*
* Tcl_FreeEncoding --
*
* This function is called to release an encoding allocated by
* Tcl_CreateEncoding() or Tcl_GetEncoding().
* Releases an encoding allocated by Tcl_CreateEncoding() or
* Tcl_GetEncoding().
*
* Results:
* None.
*
* Side effects:
* The reference count associated with the encoding is decremented and
* the encoding may be deleted if nothing is using it anymore.
* the encoding is deleted if nothing is using it anymore.
*
*---------------------------------------------------------------------------
*/
@@ -821,13 +810,14 @@ Tcl_FreeEncoding(
*
* FreeEncoding --
*
* This function is called to release an encoding by functions that
* already have the encodingMutex.
* Decrements the reference count of an encoding. The caller must hold
* encodingMutes.
*
* Results:
* None.
*
* Side effects:
* Releases the resource for an encoding if it is now unused.
* The reference count associated with the encoding is decremented and
* the encoding may be deleted if nothing is using it anymore.
*
@@ -846,15 +836,16 @@ FreeEncoding(
if (encodingPtr->refCount<=0) {
Tcl_Panic("FreeEncoding: refcount problem !!!");
}
encodingPtr->refCount--;
if (encodingPtr->refCount == 0) {
if (encodingPtr->refCount-- <= 1) {
if (encodingPtr->freeProc != NULL) {
encodingPtr->freeProc(encodingPtr->clientData);
}
if (encodingPtr->hPtr != NULL) {
Tcl_DeleteHashEntry(encodingPtr->hPtr);
}
ckfree(encodingPtr->name);
if (encodingPtr->name) {
ckfree(encodingPtr->name);
}
ckfree(encodingPtr);
}
}
@@ -925,7 +916,7 @@ Tcl_GetEncodingNames(
Tcl_MutexLock(&encodingMutex);
for (hPtr = Tcl_FirstHashEntry(&encodingTable, &search); hPtr != NULL;
hPtr = Tcl_NextHashEntry(&search)) {
Encoding *encodingPtr = Tcl_GetHashValue(hPtr);
Encoding *encodingPtr = (Encoding *)Tcl_GetHashValue(hPtr);
Tcl_CreateHashEntry(&table,
Tcl_NewStringObj(encodingPtr->name, -1), &dummy);
@@ -1017,23 +1008,22 @@ Tcl_SetSystemEncoding(
*
* Tcl_CreateEncoding --
*
* This function is called to define a new encoding and the functions
* that are used to convert between the specified encoding and Unicode.
* Defines a new encoding, along with the functions that are used to
* convert to and from Unicode.
*
* Results:
* Returns a token that represents the encoding. If an encoding with the
* same name already existed, the old encoding token remains valid and
* continues to behave as it used to, and will eventually be garbage
* collected when the last reference to it goes away. Any subsequent
* calls to Tcl_GetEncoding with the specified name will retrieve the
* most recent encoding token.
* continues to behave as it used to, and is eventually garbage collected
* when the last reference to it goes away. Any subsequent calls to
* Tcl_GetEncoding with the specified name retrieve the most recent
* encoding token.
*
* Side effects:
* The new encoding type is entered into a table visible to all
* interpreters, keyed off the encoding's name. For each call to this
* function, there should eventually be a call to Tcl_FreeEncoding, so
* that the database can be cleaned up when encodings aren't needed
* anymore.
* A new record having the name of the encoding is entered into a table of
* encodings visible to all interpreters. For each call to this function,
* there should eventually be a call to Tcl_FreeEncoding, which cleans
* deletes the record in the table when an encoding is no longer needed.
*
*---------------------------------------------------------------------------
*/
@@ -1056,13 +1046,13 @@ Tcl_CreateEncoding(
* reference goes away.
*/
encodingPtr = Tcl_GetHashValue(hPtr);
encodingPtr = (Encoding *)Tcl_GetHashValue(hPtr);
encodingPtr->hPtr = NULL;
}
name = ckalloc(strlen(typePtr->encodingName) + 1);
name = (char *)ckalloc(strlen(typePtr->encodingName) + 1);
encodingPtr = ckalloc(sizeof(Encoding));
encodingPtr = (Encoding *)ckalloc(sizeof(Encoding));
encodingPtr->name = strcpy(name, typePtr->encodingName);
encodingPtr->toUtfProc = typePtr->toUtfProc;
encodingPtr->fromUtfProc = typePtr->fromUtfProc;
@@ -1278,10 +1268,9 @@ Tcl_ExternalToUtf(
*
* Tcl_UtfToExternalDString --
*
* Convert a source buffer from UTF-8 into the specified encoding. If any
* Convert a source buffer from UTF-8 to the specified encoding. If any
* of the bytes in the source buffer are invalid or cannot be represented
* in the target encoding, a default fallback character will be
* substituted.
* in the target encoding, a default fallback character is substituted.
*
* Results:
* The converted bytes are stored in the DString, which is then NULL
@@ -1592,13 +1581,13 @@ OpenEncodingFileChannel(
* the data.
*
* Results:
* The return value is the newly loaded Encoding, or NULL if the file
* didn't exist of was in the incorrect format. If NULL was returned, an
* error message is left in interp's result object, unless interp was
* NULL.
* The return value is the newly loaded Tcl_Encoding or NULL if the file
* didn't exist or could not be processed. If NULL is returned and interp
* is not NULL, an error message is left in interp's result object.
*
* Side effects:
* File read from disk.
* A corresponding encoding file might be read from persistent storage, in
* which case LoadTableEncoding is called.
*
*---------------------------------------------------------------------------
*/
@@ -1606,8 +1595,8 @@ OpenEncodingFileChannel(
static Tcl_Encoding
LoadEncodingFile(
Tcl_Interp *interp, /* Interp for error reporting, if not NULL. */
const char *name) /* The name of the encoding file on disk and
* also the name for new encoding. */
const char *name) /* The name of both the encoding file
* and the new encoding. */
{
Tcl_Channel chan = NULL;
Tcl_Encoding encoding = NULL;
@@ -1661,27 +1650,27 @@ LoadEncodingFile(
*
* LoadTableEncoding --
*
* Helper function for LoadEncodingTable(). Loads a table to that
* converts between Unicode and some other encoding and creates an
* encoding (using a TableEncoding structure) from that information.
* Helper function for LoadEncodingFile(). Creates a Tcl_EncodingType
* structure along with its corresponding TableEncodingData structure, and
* passes it to Tcl_Createncoding.
*
* File contains binary data, but begins with a marker to indicate
* byte-ordering, so that same binary file can be read on either endian
* platforms.
* The file contains binary data but begins with a marker to indicate
* byte-ordering so a single binary file can be read on big or
* little-endian systems.
*
* Results:
* The return value is the new encoding, or NULL if the encoding could
* not be created (because the file contained invalid data).
* Returns the new Tcl_Encoding, or NULL if it could could
* not be created because the file contained invalid data.
*
* Side effects:
* None.
* See Tcl_CreateEncoding().
*
*-------------------------------------------------------------------------
*/
static Tcl_Encoding
LoadTableEncoding(
const char *name, /* Name for new encoding. */
const char *name, /* Name of the new encoding. */
int type, /* Type of encoding (ENCODING_?????). */
Tcl_Channel chan) /* File containing new encoding. */
{
@@ -1741,7 +1730,7 @@ LoadTableEncoding(
#undef PAGESIZE
#define PAGESIZE (256 * sizeof(unsigned short))
dataPtr = ckalloc(sizeof(TableEncodingData));
dataPtr = (TableEncodingData *)ckalloc(sizeof(TableEncodingData));
memset(dataPtr, 0, sizeof(TableEncodingData));
dataPtr->fallback = fallback;
@@ -1753,7 +1742,7 @@ LoadTableEncoding(
*/
size = 256 * sizeof(unsigned short *) + numPages * PAGESIZE;
dataPtr->toUnicode = ckalloc(size);
dataPtr->toUnicode = (unsigned short **)ckalloc(size);
memset(dataPtr->toUnicode, 0, size);
pageMemPtr = (unsigned short *) (dataPtr->toUnicode + 256);
@@ -1772,7 +1761,7 @@ LoadTableEncoding(
dataPtr->toUnicode[hi] = pageMemPtr;
p += 2;
for (lo = 0; lo < 256; lo++) {
if ((lo & 0x0f) == 0) {
if ((lo & 0x0F) == 0) {
p++;
}
ch = (staticHex[UCHAR(p[0])] << 12) + (staticHex[UCHAR(p[1])] << 8)
@@ -1798,10 +1787,10 @@ LoadTableEncoding(
}
/*
* Invert toUnicode array to produce the fromUnicode array. Performs a
* Invert the toUnicode array to produce the fromUnicode array. Performs a
* single malloc to get the memory for the array and all the pages needed
* by the array. While reading in the toUnicode array, we remembered what
* pages that would be needed for the fromUnicode array.
* by the array. While reading in the toUnicode array remember what
* pages are needed for the fromUnicode array.
*/
if (symbol) {
@@ -1814,7 +1803,7 @@ LoadTableEncoding(
}
}
size = 256 * sizeof(unsigned short *) + numPages * PAGESIZE;
dataPtr->fromUnicode = ckalloc(size);
dataPtr->fromUnicode = (unsigned short **)ckalloc(size);
memset(dataPtr->fromUnicode, 0, size);
pageMemPtr = (unsigned short *) (dataPtr->fromUnicode + 256);
@@ -1833,15 +1822,15 @@ LoadTableEncoding(
pageMemPtr += 256;
dataPtr->fromUnicode[ch >> 8] = page;
}
page[ch & 0xff] = (unsigned short) ((hi << 8) + lo);
page[ch & 0xFF] = (unsigned short) ((hi << 8) + lo);
}
}
}
if (type == ENCODING_MULTIBYTE) {
/*
* If multibyte encodings don't have a backslash character, define
* one. Otherwise, on Windows, native file names won't work because
* the backslash in the file name will map to the unknown character
* one. Otherwise, on Windows, native file names don't work because
* the backslash in the file name maps to the unknown character
* (question mark) when converting from UTF-8 to external encoding.
*/
@@ -1853,13 +1842,13 @@ LoadTableEncoding(
}
if (symbol) {
/*
* Make a special symbol encoding that not only maps the symbol
* characters from their Unicode code points down into page 0, but
* also ensure that the characters on page 0 map to themselves. This
* is so that a symbol font can be used to display a simple string
* like "abcd" and have alpha, beta, chi, delta show up, rather than
* have "unknown" chars show up because strictly speaking the symbol
* font doesn't have glyphs for those low ASCII chars.
* Make a special symbol encoding that maps each symbol character from
* its Unicode code point down into page 0, and also ensure that each
* characters on page 0 maps to itself so that a symbol font can be
* used to display a simple string like "abcd" and have alpha, beta,
* chi, delta show up, rather than have "unknown" chars show up because
* strictly speaking the symbol font doesn't have glyphs for those low
* ASCII chars.
*/
page = dataPtr->fromUnicode[0];
@@ -1906,7 +1895,7 @@ LoadTableEncoding(
}
/*
* Read lines from the encoding until EOF.
* Read lines until EOF.
*/
for (TclDStringClear(&lineString);
@@ -1939,7 +1928,7 @@ LoadTableEncoding(
if (from == 0) {
continue;
}
dataPtr->fromUnicode[from >> 8][from & 0xff] = to;
dataPtr->fromUnicode[from >> 8][from & 0xFF] = to;
}
}
doneParse:
@@ -1983,7 +1972,7 @@ LoadTableEncoding(
static Tcl_Encoding
LoadEscapeEncoding(
const char *name, /* Name for new encoding. */
const char *name, /* Name of the new encoding. */
Tcl_Channel chan) /* File containing new encoding. */
{
int i;
@@ -2050,9 +2039,9 @@ LoadEscapeEncoding(
Tcl_DStringFree(&lineString);
}
size = sizeof(EscapeEncodingData) - sizeof(EscapeSubTable)
size = TclOffset(EscapeEncodingData, subTables)
+ Tcl_DStringLength(&escapeData);
dataPtr = ckalloc(size);
dataPtr = (EscapeEncodingData *)ckalloc(size);
dataPtr->initLen = strlen(init);
memcpy(dataPtr->init, init, dataPtr->initLen + 1);
dataPtr->finalLen = strlen(final);
@@ -2158,7 +2147,7 @@ BinaryProc(
* UtfIntToUtfExtProc --
*
* Convert from UTF-8 to UTF-8. While converting null-bytes from the
* Tcl's internal representation (0xc0, 0x80) to the official
* Tcl's internal representation (0xC0, 0x80) to the official
* representation (0x00). See UtfToUtfProc for details.
*
* Results:
@@ -2207,7 +2196,7 @@ UtfIntToUtfExtProc(
* UtfExtToUtfIntProc --
*
* Convert from UTF-8 to UTF-8 while converting null-bytes from the
* official representation (0x00) to Tcl's internal representation (0xc0,
* official representation (0x00) to Tcl's internal representation (0xC0,
* 0x80). See UtfToUtfProc for details.
*
* Results:
@@ -2338,17 +2327,19 @@ UtfToUtfProc(
if (UCHAR(*src) < 0x80 && !(UCHAR(*src) == 0 && pureNullMode == 0)) {
/*
* Copy 7bit characters, but skip null-bytes when we are in input
* mode, so that they get converted to 0xc080.
* mode, so that they get converted to 0xC080.
*/
*dst++ = *src++;
} else if (pureNullMode == 1 && UCHAR(*src) == 0xc0 &&
*chPtr = 0; /* reset surrogate handling */
} else if (pureNullMode == 1 && UCHAR(*src) == 0xC0 &&
(src + 1 < srcEnd) && UCHAR(*(src+1)) == 0x80) {
/*
* Convert 0xc080 to real nulls when we are in output mode.
* Convert 0xC080 to real nulls when we are in output mode.
*/
*dst++ = 0;
*chPtr = 0; /* reset surrogate handling */
src += 2;
} else if (!Tcl_UtfCharComplete(src, srcEnd - src)) {
/*
@@ -2357,27 +2348,39 @@ UtfToUtfProc(
* incomplete char its bytes are made to represent themselves.
*/
*chPtr = (unsigned char) *src;
*chPtr = UCHAR(*src);
src += 1;
dst += Tcl_UniCharToUtf(*chPtr, dst);
} else {
src += TclUtfToUniChar(src, chPtr);
if ((*chPtr & 0xFC00) == 0xD800) {
/* A high surrogate character is detected, handle especially */
Tcl_UniChar low = *chPtr;
size_t len = (src <= srcEnd-3) ? Tcl_UtfToUniChar(src, &low) : 0;
if ((low & 0xFC00) != 0xDC00) {
*dst++ = (char) (((*chPtr >> 12) | 0xE0) & 0xEF);
*dst++ = (char) (((*chPtr >> 6) | 0x80) & 0xBF);
*dst++ = (char) ((*chPtr | 0x80) & 0xBF);
continue;
} else if (pureNullMode == 1) {
size_t len = TclUtfToUniChar(src, chPtr);
src += len;
if ((*chPtr & ~0x7FF) == 0xD800) {
Tcl_UniChar low;
/* A surrogate character is detected, handle especially */
#if TCL_UTF_MAX <= 4
if ((len < 3) && ((src[3 - len] & 0xC0) != 0x80)) {
/* It's invalid. See [ed29806ba] */
*chPtr = UCHAR(src[-1]);
dst += Tcl_UniCharToUtf(*chPtr, dst);
continue;
}
#endif
low = *chPtr;
len = (src <= srcEnd-3) ? Tcl_UtfToUniChar(src, &low) : 0;
if (((low & ~0x3FF) != 0xDC00) || (*chPtr & 0x400)) {
*dst++ = (char) (((*chPtr >> 12) | 0xE0) & 0xEF);
*dst++ = (char) (((*chPtr >> 6) | 0x80) & 0xBF);
*dst++ = (char) ((*chPtr | 0x80) & 0xBF);
*chPtr = 0; /* reset surrogate handling */
continue;
} else if ((TCL_UTF_MAX > 3) || (pureNullMode == 1)) {
int full = (((*chPtr & 0x3FF) << 10) | (low & 0x3FF)) + 0x10000;
*dst++ = (char) (((full >> 18) | 0xF0) & 0xF7);
*dst++ = (char) (((full >> 12) | 0x80) & 0xBF);
*dst++ = (char) (((full >> 6) | 0x80) & 0xBF);
*dst++ = (char) ((full | 0x80) & 0xBF);
*chPtr = 0;
*chPtr = 0; /* reset surrogate handling */
src += len;
continue;
}
@@ -2397,7 +2400,7 @@ UtfToUtfProc(
*
* UnicodeToUtfProc --
*
* Convert from Unicode to UTF-8.
* Convert from UTF-16 to UTF-8.
*
* Results:
* Returns TCL_OK if conversion was successful.
@@ -2410,7 +2413,7 @@ UtfToUtfProc(
static int
UnicodeToUtfProc(
ClientData clientData, /* Not used. */
ClientData clientData, /* != NULL means LE, == NUL means BE */
const char *src, /* Source string in Unicode. */
int srcLen, /* Source string length in bytes. */
int flags, /* Conversion control flags. */
@@ -2438,16 +2441,22 @@ UnicodeToUtfProc(
const char *srcStart, *srcEnd;
const char *dstEnd, *dstStart;
int result, numChars, charLimit = INT_MAX;
Tcl_UniChar ch;
unsigned short ch;
if (flags & TCL_ENCODING_CHAR_LIMIT) {
charLimit = *dstCharsPtr;
}
result = TCL_OK;
if ((srcLen % sizeof(Tcl_UniChar)) != 0) {
/* check alignment with utf-16 (2 == sizeof(UTF-16)) */
if ((srcLen % 2) != 0) {
result = TCL_CONVERT_MULTIBYTE;
srcLen /= sizeof(Tcl_UniChar);
srcLen *= sizeof(Tcl_UniChar);
srcLen--;
}
/* If last code point is a high surrogate, we cannot handle that yet */
if ((srcLen >= 2) && ((src[srcLen - (clientData?1:2)] & 0xFC) == 0xD8)) {
result = TCL_CONVERT_MULTIBYTE;
srcLen-= 2;
}
srcStart = src;
@@ -2462,18 +2471,21 @@ UnicodeToUtfProc(
break;
}
if (clientData) {
ch = (src[1] & 0xFF) << 8 | (src[0] & 0xFF);
} else {
ch = (src[0] & 0xFF) << 8 | (src[1] & 0xFF);
}
/*
* Special case for 1-byte utf chars for speed. Make sure we work with
* Tcl_UniChar-size data.
* unsigned short-size data.
*/
ch = *(Tcl_UniChar *)src;
if (ch && ch < 0x80) {
*dst++ = (ch & 0xFF);
} else {
dst += Tcl_UniCharToUtf(ch, dst);
}
src += sizeof(Tcl_UniChar);
src += sizeof(unsigned short);
}
*srcReadPtr = src - srcStart;
@@ -2487,7 +2499,7 @@ UnicodeToUtfProc(
*
* UtfToUnicodeProc --
*
* Convert from UTF-8 to Unicode.
* Convert from UTF-8 to UTF-16.
*
* Results:
* Returns TCL_OK if conversion was successful.
@@ -2500,8 +2512,7 @@ UnicodeToUtfProc(
static int
UtfToUnicodeProc(
ClientData clientData, /* TableEncodingData that specifies
* encoding. */
ClientData clientData, /* != NULL means LE, == NUL means BE */
const char *src, /* Source string in UTF-8. */
int srcLen, /* Source string length in bytes. */
int flags, /* Conversion control flags. */
@@ -2560,32 +2571,37 @@ UtfToUnicodeProc(
}
src += TclUtfToUniChar(src, chPtr);
/*
* Need to handle this in a way that won't cause misalignment by
* casting dst to a Tcl_UniChar. [Bug 1122671]
*/
#ifdef WORDS_BIGENDIAN
if (clientData) {
#if TCL_UTF_MAX > 4
*dst++ = (*chPtr >> 24);
*dst++ = ((*chPtr >> 16) & 0xFF);
*dst++ = ((*chPtr >> 8) & 0xFF);
*dst++ = (*chPtr & 0xFF);
if (*chPtr <= 0xFFFF) {
*dst++ = (*chPtr & 0xFF);
*dst++ = (*chPtr >> 8);
} else {
*dst++ = (((*chPtr - 0x10000) >> 10) & 0xFF);
*dst++ = (((*chPtr - 0x10000) >> 18) & 0x3) | 0xD8;
*dst++ = (*chPtr & 0xFF);
*dst++ = ((*chPtr & 0x3) >> 8) | 0xDC;
}
#else
*dst++ = (*chPtr >> 8);
*dst++ = (*chPtr & 0xFF);
*dst++ = (*chPtr & 0xFF);
*dst++ = (*chPtr >> 8);
#endif
#else
} else {
#if TCL_UTF_MAX > 4
*dst++ = (*chPtr & 0xFF);
*dst++ = ((*chPtr >> 8) & 0xFF);
*dst++ = ((*chPtr >> 16) & 0xFF);
*dst++ = (*chPtr >> 24);
if (*chPtr <= 0xFFFF) {
*dst++ = (*chPtr >> 8);
*dst++ = (*chPtr & 0xFF);
} else {
*dst++ = ((*chPtr & 0x3) >> 8) | 0xDC;
*dst++ = (*chPtr & 0xFF);
*dst++ = (((*chPtr - 0x10000) >> 18) & 0x3) | 0xD8;
*dst++ = (((*chPtr - 0x10000) >> 10) & 0xFF);
}
#else
*dst++ = (*chPtr & 0xFF);
*dst++ = (*chPtr >> 8);
#endif
*dst++ = (*chPtr >> 8);
*dst++ = (*chPtr & 0xFF);
#endif
}
}
*srcReadPtr = src - srcStart;
*dstWrotePtr = dst - dstStart;
@@ -2644,7 +2660,7 @@ TableToUtfProc(
Tcl_UniChar ch = 0;
const unsigned short *const *toUnicode;
const unsigned short *pageZero;
TableEncodingData *dataPtr = clientData;
TableEncodingData *dataPtr = (TableEncodingData *)clientData;
if (flags & TCL_ENCODING_CHAR_LIMIT) {
charLimit = *dstCharsPtr;
@@ -2755,7 +2771,7 @@ TableFromUtfProc(
const char *dstStart, *dstEnd, *prefixBytes;
Tcl_UniChar ch = 0;
int result, len, word, numChars;
TableEncodingData *dataPtr = clientData;
TableEncodingData *dataPtr = (TableEncodingData *)clientData;
const unsigned short *const *fromUnicode;
result = TCL_OK;
@@ -2786,12 +2802,8 @@ TableFromUtfProc(
len = TclUtfToUniChar(src, &ch);
#if TCL_UTF_MAX > 4
/*
* This prevents a crash condition. More evaluation is required for
* full support of int Tcl_UniChar. [Bug 1004065]
*/
if (ch & 0xffff0000) {
/* Unicode chars > +U0FFFF cannot be represented in any table encoding */
if (ch & 0xFFFF0000) {
word = 0;
} else
#elif TCL_UTF_MAX == 4
@@ -2799,7 +2811,7 @@ TableFromUtfProc(
word = 0;
} else
#endif
word = fromUnicode[(ch >> 8)][ch & 0xff];
word = fromUnicode[(ch >> 8)][ch & 0xFF];
if ((word == 0) && (ch != 0)) {
if (flags & TCL_ENCODING_STOPONERROR) {
@@ -2962,11 +2974,9 @@ Iso88591FromUtfProc(
{
const char *srcStart, *srcEnd, *srcClose;
const char *dstStart, *dstEnd;
int result, numChars;
int result = TCL_OK, numChars;
Tcl_UniChar ch = 0;
result = TCL_OK;
srcStart = src;
srcEnd = src + srcLen;
srcClose = srcEnd;
@@ -2995,7 +3005,7 @@ Iso88591FromUtfProc(
* Check for illegal characters.
*/
if (ch > 0xff
if (ch > 0xFF
#if TCL_UTF_MAX == 4
|| ((ch >= 0xD800) && (len < 3))
#endif
@@ -3007,7 +3017,6 @@ Iso88591FromUtfProc(
#if TCL_UTF_MAX == 4
if ((ch >= 0xD800) && (len < 3)) len = 4;
#endif
/*
* Plunge on, using '?' as a fallback character.
*/
@@ -3051,7 +3060,7 @@ TableFreeProc(
ClientData clientData) /* TableEncodingData that specifies
* encoding. */
{
TableEncodingData *dataPtr = clientData;
TableEncodingData *dataPtr = (TableEncodingData *)clientData;
/*
* Make sure we aren't freeing twice on shutdown. [Bug 219314]
@@ -3109,7 +3118,7 @@ EscapeToUtfProc(
* correspond to the bytes stored in the
* output buffer. */
{
EscapeEncodingData *dataPtr = clientData;
EscapeEncodingData *dataPtr = (EscapeEncodingData *)clientData;
const char *prefixBytes, *tablePrefixBytes, *srcStart, *srcEnd;
const unsigned short *const *tableToUnicode;
const Encoding *encodingPtr;
@@ -3120,8 +3129,8 @@ EscapeToUtfProc(
charLimit = *dstCharsPtr;
}
result = TCL_OK;
tablePrefixBytes = NULL; /* lint. */
tableToUnicode = NULL; /* lint. */
tablePrefixBytes = NULL;
tableToUnicode = NULL;
prefixBytes = dataPtr->prefixBytes;
encodingPtr = NULL;
@@ -3245,7 +3254,7 @@ EscapeToUtfProc(
TableEncodingData *tableDataPtr;
encodingPtr = GetTableEncoding(dataPtr, state);
tableDataPtr = encodingPtr->clientData;
tableDataPtr = (TableEncodingData *)encodingPtr->clientData;
tablePrefixBytes = tableDataPtr->prefixBytes;
tableToUnicode = (const unsigned short *const*)
tableDataPtr->toUnicode;
@@ -3323,7 +3332,7 @@ EscapeFromUtfProc(
* correspond to the bytes stored in the
* output buffer. */
{
EscapeEncodingData *dataPtr = clientData;
EscapeEncodingData *dataPtr = (EscapeEncodingData *)clientData;
const Encoding *encodingPtr;
const char *srcStart, *srcEnd, *srcClose;
const char *dstStart, *dstEnd;
@@ -3364,7 +3373,7 @@ EscapeFromUtfProc(
}
encodingPtr = GetTableEncoding(dataPtr, state);
tableDataPtr = encodingPtr->clientData;
tableDataPtr = (const TableEncodingData *)encodingPtr->clientData;
tablePrefixBytes = tableDataPtr->prefixBytes;
tableFromUnicode = (const unsigned short *const *)
tableDataPtr->fromUnicode;
@@ -3383,7 +3392,7 @@ EscapeFromUtfProc(
break;
}
len = TclUtfToUniChar(src, &ch);
word = tableFromUnicode[(ch >> 8)][ch & 0xff];
word = tableFromUnicode[(ch >> 8)][ch & 0xFF];
if ((word == 0) && (ch != 0)) {
int oldState;
@@ -3392,8 +3401,8 @@ EscapeFromUtfProc(
oldState = state;
for (state = 0; state < dataPtr->numSubTables; state++) {
encodingPtr = GetTableEncoding(dataPtr, state);
tableDataPtr = encodingPtr->clientData;
word = tableDataPtr->fromUnicode[(ch >> 8)][ch & 0xff];
tableDataPtr = (const TableEncodingData *)encodingPtr->clientData;
word = tableDataPtr->fromUnicode[(ch >> 8)][ch & 0xFF];
if (word != 0) {
break;
}
@@ -3406,7 +3415,7 @@ EscapeFromUtfProc(
break;
}
encodingPtr = GetTableEncoding(dataPtr, state);
tableDataPtr = encodingPtr->clientData;
tableDataPtr = (const TableEncodingData *)encodingPtr->clientData;
word = tableDataPtr->fallback;
}
@@ -3496,14 +3505,13 @@ EscapeFromUtfProc(
*
* EscapeFreeProc --
*
* This function is invoked when an EscapeEncodingData encoding is
* deleted. It deletes the memory used by the encoding.
* Frees resources used by the encoding.
*
* Results:
* None.
*
* Side effects:
* Memory freed.
* Memory is freed.
*
*---------------------------------------------------------------------------
*/
@@ -3513,7 +3521,7 @@ EscapeFreeProc(
ClientData clientData) /* EscapeEncodingData that specifies
* encoding. */
{
EscapeEncodingData *dataPtr = clientData;
EscapeEncodingData *dataPtr = (EscapeEncodingData *)clientData;
EscapeSubTable *subTablePtr;
int i;
@@ -3678,8 +3686,8 @@ InitializeEncodingSearchPath(
bytes = Tcl_GetStringFromObj(searchPathObj, &numBytes);
*lengthPtr = numBytes;
*valuePtr = ckalloc(numBytes + 1);
memcpy(*valuePtr, bytes, (size_t) numBytes + 1);
*valuePtr = (char *)ckalloc(numBytes + 1);
memcpy(*valuePtr, bytes, numBytes + 1);
Tcl_DecrRefCount(searchPathObj);
}