Import Tcl-code 8.6.8

tests/encoding.test

@@ -34,6 +34,7 @@ proc runtests {} {
 
 # Some tests require the testencoding command
 testConstraint testencoding [llength [info commands testencoding]]
+testConstraint fullutf [expr {[format %c 0x010000] != "\ufffd"}]
 testConstraint exec [llength [info commands exec]]
 testConstraint testgetdefenc [llength [info commands testgetdefenc]]
 
@@ -73,12 +74,12 @@ test encoding-2.2 {Tcl_FreeEncoding: refcount != 0} -setup {
 } -constraints {testencoding} -body {
     encoding system shiftjis		;# incr ref count
     encoding dirs [list [pwd]]
-    set x [encoding convertto shiftjis \u4e4e]	;# old one found   
-    encoding system identity
+    set x [encoding convertto shiftjis \u4e4e]	;# old one found
+    encoding system iso8859-1
     llength shiftjis		;# Shimmer away any cache of Tcl_Encoding
     lappend x [catch {encoding convertto shiftjis \u4e4e} msg] $msg
 } -cleanup {
-    encoding system identity
+    encoding system iso8859-1
     encoding dirs $path
     encoding system $system
 } -result "\u008c\u00c1 1 {unknown encoding \"shiftjis\"}"
@@ -135,7 +136,7 @@ test encoding-5.1 {Tcl_SetSystemEncoding} -setup {
     encoding system jis0208
     encoding convertto \u4e4e
 } -cleanup {
-    encoding system identity
+    encoding system iso8859-1
     encoding system $old
 } -result {8C}
 test encoding-5.2 {Tcl_SetSystemEncoding: test ref count} {
@@ -182,7 +183,7 @@ test encoding-8.1 {Tcl_ExternalToUtf} {
     puts -nonewline $f "ab\x8c\xc1g"
     close $f
     set f [open [file join [temporaryDirectory] dummy] r]
-    fconfigure $f -translation binary -encoding shiftjis    
+    fconfigure $f -translation binary -encoding shiftjis
     set x [read $f]
     close $f
     file delete [file join [temporaryDirectory] dummy]
@@ -258,14 +259,14 @@ test encoding-11.5.1 {LoadEncodingFile: escape file} {
 test encoding-11.6 {LoadEncodingFile: invalid file} -constraints {testencoding} -setup {
     set system [encoding system]
     set path [encoding dirs]
-    encoding system identity
+    encoding system iso8859-1
 } -body {
     cd [temporaryDirectory]
     encoding dirs [file join tmp encoding]
     makeDirectory tmp
     makeDirectory [file join tmp encoding]
     set f [open [file join tmp encoding splat.enc] w]
-    fconfigure $f -translation binary 
+    fconfigure $f -translation binary
     puts $f "abcdefghijklmnop"
     close $f
     encoding convertto splat \u4e4e
@@ -286,11 +287,11 @@ test encoding-12.1 {LoadTableEncoding: normal encoding} {
     append x [encoding convertfrom iso8859-3 \xd5]
 } "\xd5?\u120"
 test encoding-12.2 {LoadTableEncoding: single-byte encoding} {
-    set x [encoding convertto iso8859-3 ab\u0120g] 
+    set x [encoding convertto iso8859-3 ab\u0120g]
     append x [encoding convertfrom iso8859-3 ab\xd5g]
 } "ab\xd5gab\u120g"
 test encoding-12.3 {LoadTableEncoding: multi-byte encoding} {
-    set x [encoding convertto shiftjis ab\u4e4eg] 
+    set x [encoding convertto shiftjis ab\u4e4eg]
     append x [encoding convertfrom shiftjis ab\x8c\xc1g]
 } "ab\x8c\xc1gab\u4e4eg"
 test encoding-12.4 {LoadTableEncoding: double-byte encoding} {
@@ -332,9 +333,14 @@ test encoding-16.1 {UnicodeToUtfProc} {
     set val [encoding convertfrom unicode NN]
     list $val [format %x [scan $val %c]]
 } "\u4e4e 4e4e"
+test encoding-16.2 {UnicodeToUtfProc} -constraints fullutf -body {
+    set val [encoding convertfrom unicode "\xd8\xd8\xdc\xdc"]
+    list $val [format %x [scan $val %c]]
+} -result "\U460dc 460dc"
 
-test encoding-17.1 {UtfToUnicodeProc} {
-} {}
+test encoding-17.1 {UtfToUnicodeProc} -constraints fullutf -body {
+    encoding convertto unicode "\U460dc"
+} -result "\xd8\xd8\xdc\xdc"
 
 test encoding-18.1 {TableToUtfProc} {
 } {}
@@ -448,6 +454,31 @@ test encoding-24.3 {EscapeFreeProc on open channels} {stdio} {
     list $count [viewable $line]
 } [list 3 "\u4e4e\u4e5e\u4e5f (\\u4e4e\\u4e5e\\u4e5f)"]
 
+test encoding-24.4 {Parse valid or invalid utf-8} {
+    string length [encoding convertfrom utf-8 "\xc0\x80"]
+} 1
+test encoding-24.5 {Parse valid or invalid utf-8} {
+    string length [encoding convertfrom utf-8 "\xc0\x81"]
+} 2
+test encoding-24.6 {Parse valid or invalid utf-8} {
+    string length [encoding convertfrom utf-8 "\xc1\xbf"]
+} 2
+test encoding-24.7 {Parse valid or invalid utf-8} {
+    string length [encoding convertfrom utf-8 "\xc2\x80"]
+} 1
+test encoding-24.8 {Parse valid or invalid utf-8} {
+    string length [encoding convertfrom utf-8 "\xe0\x80\x80"]
+} 3
+test encoding-24.9 {Parse valid or invalid utf-8} {
+    string length [encoding convertfrom utf-8 "\xe0\x9f\xbf"]
+} 3
+test encoding-24.10 {Parse valid or invalid utf-8} {
+    string length [encoding convertfrom utf-8 "\xe0\xa0\x80"]
+} 1
+test encoding-24.11 {Parse valid or invalid utf-8} {
+    string length [encoding convertfrom utf-8 "\xef\xbf\xbf"]
+} 1
+
 file delete [file join [temporaryDirectory] iso2022.txt]
 
 #