/usr/share/doc/perl-doc-html/perluniintro.html is in perl-doc-html 5.22.0-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 | <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<title>perluniintro - perldoc.perl.org</title>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
<meta http-equiv="Content-Language" content="en-gb">
<link rel="search" type="application/opensearchdescription+xml" title="Search perldoc.perl.org" href="/static/search.xml"/>
<link href="static/css-20100830.css" rel="stylesheet" rev="stylesheet" type="text/css" media="screen">
<link href="static/exploreperl.css" rel="stylesheet" rev="stylesheet" type="text/css">
</head>
<body onLoad="perldoc.startup();" onPageShow="if (event.persisted) perldoc.startup();">
<div id="page">
<div id="header">
<div id="homepage_link">
<a href="index.html"></a>
</div>
<div id="strapline">
Perl Programming Documentation
</div>
<div id="download_link" class="download">
<a href="http://www.perl.org/get.html">Download Perl</a>
</div>
<div id="explore_link" class="download">
<a id="explore_anchor" href="#">Explore</a>
</div>
</div>
<div id="body">
<div id="left_column">
<div class="side_group">
<div class="side_panel doc_panel">
<p>Manual</p>
<ul>
<li><a href="index-overview.html">Overview</a>
<li><a href="index-tutorials.html">Tutorials</a>
<li><a href="index-faq.html">FAQs</a>
<li><a href="index-history.html">History / Changes</a>
<li><a href="index-licence.html">License</a>
</ul>
</div>
<div class="side_panel doc_panel">
<p>Reference</p>
<ul>
<li><a href="index-language.html">Language</a>
<li><a href="index-functions.html">Functions</a>
<li><a href="perlop.html">Operators</a>
<li><a href="perlvar.html">Special Variables</a>
<li><a href="index-pragmas.html">Pragmas</a>
<li><a href="index-utilities.html">Utilities</a>
<li><a href="index-internals.html">Internals</a>
<li><a href="index-platforms.html">Platform Specific</a>
</ul>
</div>
<div class="side_panel doc_panel">
<p>Modules</p>
<ul>
<li>
<a href="index-modules-A.html">A</a>
•
<a href="index-modules-B.html">B</a>
•
<a href="index-modules-C.html">C</a>
•
<a href="index-modules-D.html">D</a>
•
<a href="index-modules-E.html">E</a>
<li>
<a href="index-modules-F.html">F</a>
•
<a href="index-modules-G.html">G</a>
•
<a href="index-modules-H.html">H</a>
•
<a href="index-modules-I.html">I</a>
•
<a href="index-modules-L.html">L</a>
<li>
<a href="index-modules-M.html">M</a>
•
<a href="index-modules-N.html">N</a>
•
<a href="index-modules-O.html">O</a>
•
<a href="index-modules-P.html">P</a>
•
<a href="index-modules-S.html">S</a>
<li>
<a href="index-modules-T.html">T</a>
•
<a href="index-modules-U.html">U</a>
•
<a href="index-modules-X.html">X</a>
</ul>
</div>
<div class="side_panel doc_panel">
<p>Tools</p>
<ul>
<li><a href="preferences.html">Preferences</a>
</ul>
</div>
</div>
</div>
<div id="centre_column">
<div id="content_header">
<div id="title_bar">
<div id="page_name">
<h1>perluniintro</h1>
</div>
<div id="perl_version">
Perl 5 version 22.0 documentation
</div>
<div class="page_links" id="page_links_top">
<a href="#" onClick="toolbar.goToTop();return false;">Go to top</a>
</div>
<div class="page_links" id="page_links_bottom">
<a href="#" id="page_index_toggle">Show page index</a> •
<a href="#" id="recent_pages_toggle">Show recent pages</a>
</div>
<div id="search_form">
<form action="search.html" method="GET" id="search">
<input type="text" name="q" id="search_box" alt="Search">
</form>
</div>
</div>
<div id="breadcrumbs">
<a href="index.html">Home</a> >
<a href="index-language.html">Language reference</a> >
perluniintro
</div>
</div>
<div id="content_body">
<!--[if lt IE 7]>
<div class="noscript">
<p>
<strong>It looks like you're using Internet Explorer 6. This is a very old
browser which does not offer full support for modern websites.</strong>
</p>
<p>
Unfortunately this means that this website will not work on
your computer.
</p>
<p>
Don't miss out though! To view the site (and get a better experience from
many other websites), simply upgrade to
<a href="http://www.microsoft.com/windows/Internet-explorer/default.aspx">Internet
Explorer 8</a>
or download an alternative browser such as
<a href="http://www.mozilla.com/en-US/firefox/firefox.html">Firefox</a>,
<a href="http://www.apple.com/safari/download/">Safari</a>, or
<a href="http://www.google.co.uk/chrome">Google Chrome</a>.
</p>
<p>
All of these browsers are free. If you're using a PC at work, you may
need to contact your IT administrator.
</p>
</div>
<![endif]-->
<noscript>
<div class="noscript">
<p>
<strong>Please note: Many features of this site require JavaScript. You appear to have JavaScript disabled,
or are running a non-JavaScript capable web browser.</strong>
</p>
<p>
To get the best experience, please enable JavaScript or download a modern web browser such as <a href="http://www.microsoft.com/windows/Internet-explorer/default.aspx">Internet Explorer 8</a>, <a href="http://www.mozilla.com/en-US/firefox/firefox.html">Firefox</a>, <a href="http://www.apple.com/safari/download/">Safari</a>, or <a href="http://www.google.co.uk/chrome">Google Chrome</a>.
</p>
</div>
</noscript>
<div id="recent_pages" class="hud_container">
<div id="recent_pages_header" class="hud_header">
<div id="recent_pages_close" class="hud_close"><a href="#" onClick="recentPages.hide();return false;"></a></div>
<div id="recent_pages_title" class="hud_title"><span class="hud_span_top">Recently read</span></div>
<div id="recent_pages_topright" class="hud_topright"></div>
</div>
<div id="recent_pages_content" class="hud_content">
</div>
<div id="recent_pages_footer" class="hud_footer">
<div id="recent_pages_bottomleft" class="hud_bottomleft"></div>
<div id="recent_pages_bottom" class="hud_bottom"><span class="hud_span_bottom"></span></div>
<div id="recent_pages_resize" class="hud_resize"></div>
</div>
</div>
<div id="from_search"></div>
<h1>perluniintro</h1>
<!-- -->
<ul><li><a href="#NAME">NAME</a><li><a href="#DESCRIPTION">DESCRIPTION</a><ul><li><a href="#Unicode">Unicode</a><li><a href="#Perl's-Unicode-Support">Perl's Unicode Support</a><li><a href="#Perl's-Unicode-Model">Perl's Unicode Model</a><li><a href="#Unicode-and-EBCDIC">Unicode and EBCDIC</a><li><a href="#Creating-Unicode">Creating Unicode</a><li><a href="#Handling-Unicode">Handling Unicode</a><li><a href="#Legacy-Encodings">Legacy Encodings</a><li><a href="#Unicode-I%2fO">Unicode I/O</a><li><a href="#Displaying-Unicode-As-Text">Displaying Unicode As Text</a><li><a href="#Special-Cases">Special Cases</a><li><a href="#Advanced-Topics">Advanced Topics</a><li><a href="#Miscellaneous">Miscellaneous</a><li><a href="#Questions-With-Answers">Questions With Answers</a><li><a href="#Hexadecimal-Notation">Hexadecimal Notation</a><li><a href="#Further-Resources">Further Resources</a></ul><li><a href="#UNICODE-IN-OLDER-PERLS">UNICODE IN OLDER PERLS</a><li><a href="#SEE-ALSO">SEE ALSO</a><li><a href="#ACKNOWLEDGMENTS">ACKNOWLEDGMENTS</a><li><a href="#AUTHOR%2c-COPYRIGHT%2c-AND-LICENSE">AUTHOR, COPYRIGHT, AND LICENSE</a></ul><a name="NAME"></a><h1>NAME</h1>
<p>perluniintro - Perl Unicode introduction</p>
<a name="DESCRIPTION"></a><h1>DESCRIPTION</h1>
<p>This document gives a general idea of Unicode and how to use Unicode
in Perl. See <a href="#Further-Resources">Further Resources</a> for references to more in-depth
treatments of Unicode.</p>
<a name="Unicode"></a><h2>Unicode</h2>
<p>Unicode is a character set standard which plans to codify all of the
writing systems of the world, plus many other symbols.</p>
<p>Unicode and ISO/IEC 10646 are coordinated standards that unify
almost all other modern character set standards,
covering more than 80 writing systems and hundreds of languages,
including all commercially-important modern languages. All characters
in the largest Chinese, Japanese, and Korean dictionaries are also
encoded. The standards will eventually cover almost all characters in
more than 250 writing systems and thousands of languages.
Unicode 1.0 was released in October 1991, and 6.0 in October 2010.</p>
<p>A Unicode <i>character</i> is an abstract entity. It is not bound to any
particular integer width, especially not to the C language <code class="inline"><span class="w">char</span></code>
.
Unicode is language-neutral and display-neutral: it does not encode the
language of the text, and it does not generally define fonts or other graphical
layout details. Unicode operates on characters and on text built from
those characters.</p>
<p>Unicode defines characters like <code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span></code>
or <code class="inline"><span class="w">GREEK</span>
<span class="w">SMALL</span> <span class="w">LETTER</span> <span class="w">ALPHA</span></code>
and unique numbers for the characters, in this
case 0x0041 and 0x03B1, respectively. These unique numbers are called
<i>code points</i>. A code point is essentially the position of the
character within the set of all possible Unicode characters, and thus in
Perl, the term <i>ordinal</i> is often used interchangeably with it.</p>
<p>The Unicode standard prefers using hexadecimal notation for the code
points. If numbers like <code class="inline"><span class="n">0x0041</span></code>
are unfamiliar to you, take a peek
at a later section, <a href="#Hexadecimal-Notation">Hexadecimal Notation</a>. The Unicode standard
uses the notation <code class="inline">U+0041 LATIN CAPITAL LETTER A</code>, to give the
hexadecimal code point and the normative name of the character.</p>
<p>Unicode also defines various <i>properties</i> for the characters, like
"uppercase" or "lowercase", "decimal digit", or "punctuation";
these properties are independent of the names of the characters.
Furthermore, various operations on the characters like uppercasing,
lowercasing, and collating (sorting) are defined.</p>
<p>A Unicode <i>logical</i> "character" can actually consist of more than one internal
<i>actual</i> "character" or code point. For Western languages, this is adequately
modelled by a <i>base character</i> (like <code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span></code>
) followed
by one or more <i>modifiers</i> (like <code class="inline"><span class="w">COMBINING</span> <span class="w">ACUTE</span> <span class="w">ACCENT</span></code>
). This sequence of
base character and modifiers is called a <i>combining character
sequence</i>. Some non-western languages require more complicated
models, so Unicode created the <i>grapheme cluster</i> concept, which was
later further refined into the <i>extended grapheme cluster</i>. For
example, a Korean Hangul syllable is considered a single logical
character, but most often consists of three actual
Unicode characters: a leading consonant followed by an interior vowel followed
by a trailing consonant.</p>
<p>Whether to call these extended grapheme clusters "characters" depends on your
point of view. If you are a programmer, you probably would tend towards seeing
each element in the sequences as one unit, or "character". However from
the user's point of view, the whole sequence could be seen as one
"character" since that's probably what it looks like in the context of the
user's language. In this document, we take the programmer's point of
view: one "character" is one Unicode code point.</p>
<p>For some combinations of base character and modifiers, there are
<i>precomposed</i> characters. There is a single character equivalent, for
example, for the sequence <code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span></code>
followed by
<code class="inline"><span class="w">COMBINING</span> <span class="w">ACUTE</span> <span class="w">ACCENT</span></code>
. It is called <code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span> <span class="w">WITH</span>
<span class="w">ACUTE</span></code>
. These precomposed characters are, however, only available for
some combinations, and are mainly meant to support round-trip
conversions between Unicode and legacy standards (like ISO 8859). Using
sequences, as Unicode does, allows for needing fewer basic building blocks
(code points) to express many more potential grapheme clusters. To
support conversion between equivalent forms, various <i>normalization
forms</i> are also defined. Thus, <code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span> <span class="w">WITH</span> <span class="w">ACUTE</span></code>
is
in <i>Normalization Form Composed</i>, (abbreviated NFC), and the sequence
<code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span></code>
followed by <code class="inline"><span class="w">COMBINING</span> <span class="w">ACUTE</span> <span class="w">ACCENT</span></code>
represents the same character in <i>Normalization Form Decomposed</i> (NFD).</p>
<p>Because of backward compatibility with legacy encodings, the "a unique
number for every character" idea breaks down a bit: instead, there is
"at least one number for every character". The same character could
be represented differently in several legacy encodings. The
converse is not true: some code points do not have an assigned
character. Firstly, there are unallocated code points within
otherwise used blocks. Secondly, there are special Unicode control
characters that do not represent true characters.</p>
<p>When Unicode was first conceived, it was thought that all the world's
characters could be represented using a 16-bit word; that is a maximum of
<code class="inline"><span class="n">0x10000</span></code>
(or 65,536) characters would be needed, from <code class="inline"><span class="n">0x0000</span></code>
to
<code class="inline"><span class="n">0xFFFF</span></code>
. This soon proved to be wrong, and since Unicode 2.0 (July
1996), Unicode has been defined all the way up to 21 bits (<code class="inline"><span class="n">0x10FFFF</span></code>
),
and Unicode 3.1 (March 2001) defined the first characters above <code class="inline"><span class="n">0xFFFF</span></code>
.
The first <code class="inline"><span class="n">0x10000</span></code>
characters are called the <i>Plane 0</i>, or the
<i>Basic Multilingual Plane</i> (BMP). With Unicode 3.1, 17 (yes,
seventeen) planes in all were defined--but they are nowhere near full of
defined characters, yet.</p>
<p>When a new language is being encoded, Unicode generally will choose a
<code class="inline"><span class="w">block</span></code>
of consecutive unallocated code points for its characters. So
far, the number of code points in these blocks has always been evenly
divisible by 16. Extras in a block, not currently needed, are left
unallocated, for future growth. But there have been occasions when
a later release needed more code points than the available extras, and a
new block had to allocated somewhere else, not contiguous to the initial
one, to handle the overflow. Thus, it became apparent early on that
"block" wasn't an adequate organizing principal, and so the <code class="inline"><span class="w">Script</span></code>
property was created. (Later an improved script property was added as
well, the <code class="inline"><span class="w">Script_Extensions</span></code>
property.) Those code points that are in
overflow blocks can still
have the same script as the original ones. The script concept fits more
closely with natural language: there is <code class="inline"><span class="w">Latin</span></code>
script, <code class="inline"><span class="w">Greek</span></code>
script, and so on; and there are several artificial scripts, like
<code class="inline"><span class="w">Common</span></code>
for characters that are used in multiple scripts, such as
mathematical symbols. Scripts usually span varied parts of several
blocks. For more information about scripts, see <a href="perlunicode.html#Scripts">Scripts in perlunicode</a>.
The division into blocks exists, but it is almost completely
accidental--an artifact of how the characters have been and still are
allocated. (Note that this paragraph has oversimplified things for the
sake of this being an introduction. Unicode doesn't really encode
languages, but the writing systems for them--their scripts; and one
script can be used by many languages. Unicode also encodes things that
aren't really about languages, such as symbols like <code class="inline"><span class="w">BAGGAGE</span> <span class="w">CLAIM</span></code>
.)</p>
<p>The Unicode code points are just abstract numbers. To input and
output these abstract numbers, the numbers must be <i>encoded</i> or
<i>serialised</i> somehow. Unicode defines several <i>character encoding
forms</i>, of which <i>UTF-8</i> is the most popular. UTF-8 is a
variable length encoding that encodes Unicode characters as 1 to 4
bytes. Other encodings
include UTF-16 and UTF-32 and their big- and little-endian variants
(UTF-8 is byte-order independent). The ISO/IEC 10646 defines the UCS-2
and UCS-4 encoding forms.</p>
<p>For more information about encodings--for instance, to learn what
<i>surrogates</i> and <i>byte order marks</i> (BOMs) are--see <a href="perlunicode.html">perlunicode</a>.</p>
<a name="Perl's-Unicode-Support"></a><h2>Perl's Unicode Support</h2>
<p>Starting from Perl v5.6.0, Perl has had the capacity to handle Unicode
natively. Perl v5.8.0, however, is the first recommended release for
serious Unicode work. The maintenance release 5.6.1 fixed many of the
problems of the initial Unicode implementation, but for example
regular expressions still do not work with Unicode in 5.6.1.
Perl v5.14.0 is the first release where Unicode support is
(almost) seamlessly integrable without some gotchas (the exception being
some differences in <a href="functions/quotemeta.html">quotemeta</a>, and that is fixed
starting in Perl 5.16.0). To enable this
seamless support, you should <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">feature</span> <span class="q">'unicode_strings'</span></code>
(which is
automatically selected if you <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="n">5.012</span></code>
or higher). See <a href="feature.html">feature</a>.
(5.14 also fixes a number of bugs and departures from the Unicode
standard.)</p>
<p>Before Perl v5.8.0, the use of <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">utf8</span></code>
was used to declare
that operations in the current block or file would be Unicode-aware.
This model was found to be wrong, or at least clumsy: the "Unicodeness"
is now carried with the data, instead of being attached to the
operations.
Starting with Perl v5.8.0, only one case remains where an explicit <code class="inline"><a class="l_k" href="functions/use.html">use</a>
<span class="w">utf8</span></code>
is needed: if your Perl script itself is encoded in UTF-8, you can
use UTF-8 in your identifier names, and in string and regular expression
literals, by saying <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">utf8</span></code>
. This is not the default because
scripts with legacy 8-bit data in them would break. See <a href="utf8.html">utf8</a>.</p>
<a name="Perl's-Unicode-Model"></a><h2>Perl's Unicode Model</h2>
<p>Perl supports both pre-5.6 strings of eight-bit native bytes, and
strings of Unicode characters. The general principle is that Perl tries
to keep its data as eight-bit bytes for as long as possible, but as soon
as Unicodeness cannot be avoided, the data is transparently upgraded
to Unicode. Prior to Perl v5.14.0, the upgrade was not completely
transparent (see <a href="perlunicode.html#The-%22Unicode-Bug%22">The Unicode Bug in perlunicode</a>), and for backwards
compatibility, full transparency is not gained unless <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">feature</span>
<span class="q">'unicode_strings'</span></code>
(see <a href="feature.html">feature</a>) or <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="n">5.012</span></code>
(or higher) is
selected.</p>
<p>Internally, Perl currently uses either whatever the native eight-bit
character set of the platform (for example Latin-1) is, defaulting to
UTF-8, to encode Unicode strings. Specifically, if all code points in
the string are <code class="inline"><span class="n">0xFF</span></code>
or less, Perl uses the native eight-bit
character set. Otherwise, it uses UTF-8.</p>
<p>A user of Perl does not normally need to know nor care how Perl
happens to encode its internal strings, but it becomes relevant when
outputting Unicode strings to a stream without a PerlIO layer (one with
the "default" encoding). In such a case, the raw bytes used internally
(the native character set or UTF-8, as appropriate for each string)
will be used, and a "Wide character" warning will be issued if those
strings contain a character beyond 0x00FF.</p>
<p>For example,</p>
<pre class="verbatim"><ol><li> <span class="w">perl</span> -e <span class="q">'print "\x{DF}\n", "\x{0100}\x{DF}\n"'</span></li></ol></pre><p>produces a fairly useless mixture of native bytes and UTF-8, as well
as a warning:</p>
<pre class="verbatim"><ol><li> <span class="w">Wide</span> <span class="w">character</span> <span class="w">in</span> <a class="l_k" href="functions/print.html">print</a> <span class="i">at</span> ...</li></ol></pre><p>To output UTF-8, use the <code class="inline"><span class="j">:</span><span class="w">encoding</span></code>
or <code class="inline"><span class="j">:</span><span class="w">utf8</span></code>
output layer. Prepending</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/binmode.html">binmode</a><span class="s">(</span><span class="w">STDOUT</span><span class="cm">,</span> <span class="q">":utf8"</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>to this sample program ensures that the output is completely UTF-8,
and removes the program's warning.</p>
<p>You can enable automatic UTF-8-ification of your standard file
handles, default <code class="inline"><a class="l_k" href="functions/open.html">open()</a></code> layer, and <code class="inline"><span class="i">@ARGV</span></code>
by using either
the <code class="inline">-C</code>
command line switch or the <code class="inline"><span class="w">PERL_UNICODE</span></code>
environment
variable, see <a href="perlrun.html">perlrun</a> for the documentation of the <code class="inline">-C</code>
switch.</p>
<p>Note that this means that Perl expects other software to work the same
way:
if Perl has been led to believe that STDIN should be UTF-8, but then
STDIN coming in from another command is not UTF-8, Perl will likely
complain about the malformed UTF-8.</p>
<p>All features that combine Unicode and I/O also require using the new
PerlIO feature. Almost all Perl 5.8 platforms do use PerlIO, though:
you can see whether yours is by running "perl -V" and looking for
<code class="inline"><span class="w">useperlio</span>=<span class="w">define</span></code>
.</p>
<a name="Unicode-and-EBCDIC"></a><h2>Unicode and EBCDIC</h2>
<p>Perl 5.8.0 added support for Unicode on EBCDIC platforms. This support
was allowed to lapse in later releases, but was revived in 5.22.
Unicode support is somewhat more complex to implement since additional
conversions are needed. See <a href="perlebcdic.html">perlebcdic</a> for more information.</p>
<p>On EBCDIC platforms, the internal Unicode encoding form is UTF-EBCDIC
instead of UTF-8. The difference is that as UTF-8 is "ASCII-safe" in
that ASCII characters encode to UTF-8 as-is, while UTF-EBCDIC is
"EBCDIC-safe", in that all the basic characters (which includes all
those that have ASCII equivalents (like <code class="inline"><span class="q">"A"</span></code>
, <code class="inline"><span class="q">"0"</span></code>
, <code class="inline"><span class="q">"%"</span></code>
, <i>etc.</i>)
are the same in both EBCDIC and UTF-EBCDIC. Often, documentation
will use the term "UTF-8" to mean UTF-EBCDIC as well. This is the case
in this document.</p>
<a name="Creating-Unicode"></a><h2>Creating Unicode</h2>
<p>This section applies fully to Perls starting with v5.22. Various
caveats for earlier releases are in the <a href="#Earlier-releases-caveats">Earlier releases caveats</a>
subsection below.</p>
<p>To create Unicode characters in literals,
use the <code class="inline">\<span class="i">N</span><span class="s">{</span>...<span class="s">}</span></code>
notation in double-quoted strings:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$smiley_from_name</span> = <span class="q">"\N{WHITE SMILING FACE}"</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$smiley_from_code_point</span> = <span class="q">"\N{U+263a}"</span><span class="sc">;</span></li></ol></pre><p>Similarly, they can be used in regular expression literals</p>
<pre class="verbatim"><ol><li> <span class="i">$smiley</span> =~ <span class="q">/\N{WHITE SMILING FACE}/</span><span class="sc">;</span></li><li> <span class="i">$smiley</span> =~ <span class="q">/\N{U+263a}/</span><span class="sc">;</span></li></ol></pre><p>At run-time you can use:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/use.html">use</a> <span class="w">charnames</span> <span class="s">(</span><span class="s">)</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$hebrew_alef_from_name</span></li><li> = <span class="i">charnames::string_vianame</span><span class="s">(</span><span class="q">"HEBREW LETTER ALEF"</span><span class="s">)</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$hebrew_alef_from_code_point</span> = <span class="i">charnames::string_vianame</span><span class="s">(</span><span class="q">"U+05D0"</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>Naturally, <code class="inline"><a class="l_k" href="functions/ord.html">ord()</a></code> will do the reverse: it turns a character into
a code point.</p>
<p>There are other runtime options as well. You can use <code class="inline"><a class="l_k" href="functions/pack.html">pack()</a></code>:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$hebrew_alef_from_code_point</span> = <a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">"U"</span><span class="cm">,</span> <span class="n">0x05d0</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>Or you can use <code class="inline"><a class="l_k" href="functions/chr.html">chr()</a></code>, though it is less convenient in the general
case:</p>
<pre class="verbatim"><ol><li> <span class="i">$hebrew_alef_from_code_point</span> = <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="i">utf8::unicode_to_native</span><span class="s">(</span><span class="n">0x05d0</span><span class="s">)</span><span class="s">)</span><span class="sc">;</span></li><li> <span class="i">utf8::upgrade</span><span class="s">(</span><span class="i">$hebrew_alef_from_code_point</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>The <code class="inline"><span class="i">utf8::unicode_to_native</span><span class="s">(</span><span class="s">)</span></code>
and <code class="inline"><span class="i">utf8::upgrade</span><span class="s">(</span><span class="s">)</span></code>
aren't needed if
the argument is above 0xFF, so the above could have been written as</p>
<pre class="verbatim"><ol><li> <span class="i">$hebrew_alef_from_code_point</span> = <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="n">0x05d0</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>since 0x5d0 is above 255.</p>
<p><code class="inline">\<span class="w">x</span><span class="s">{</span><span class="s">}</span></code>
and <code class="inline">\<span class="w">o</span><span class="s">{</span><span class="s">}</span></code>
can also be used to specify code points at compile
time in double-quotish strings, but, for backward compatibility with
older Perls, the same rules apply as with <code class="inline"><a class="l_k" href="functions/chr.html">chr()</a></code> for code points less
than 256.</p>
<p><code class="inline"><span class="i">utf8::unicode_to_native</span><span class="s">(</span><span class="s">)</span></code>
is used so that the Perl code is portable
to EBCDIC platforms. You can omit it if you're <i>really</i> sure no one
will ever want to use your code on a non-ASCII platform. Starting in
Perl v5.22, calls to it on ASCII platforms are optimized out, so there's
no performance penalty at all in adding it. Or you can simply use the
other constructs that don't require it.</p>
<p>See <a href="#Further-Resources">Further Resources</a> for how to find all these names and numeric
codes.</p>
<a name="Earlier-releases-caveats"></a><h3>Earlier releases caveats</h3>
<p>On EBCDIC platforms, prior to v5.22, using <code class="inline">\<span class="i">N</span><span class="s">{</span><span class="w">U</span>+...<span class="s">}</span></code>
doesn't work
properly.</p>
<p>Prior to v5.16, using <code class="inline">\<span class="i">N</span><span class="s">{</span>...<span class="s">}</span></code>
with a character name (as opposed to a
<code class="inline"><span class="w">U</span>+...</code>
code point) required a <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">charnames</span> <span class="co">:</span><span class="w">full</span></code>
.</p>
<p>Prior to v5.14, there were some bugs in <code class="inline">\<span class="i">N</span><span class="s">{</span>...<span class="s">}</span></code>
with a character name
(as opposed to a <code class="inline"><span class="w">U</span>+...</code>
code point).</p>
<p><code class="inline"><span class="i">charnames::string_vianame</span><span class="s">(</span><span class="s">)</span></code>
was introduced in v5.14. Prior to that,
<code class="inline"><span class="i">charnames::vianame</span><span class="s">(</span><span class="s">)</span></code>
should work, but only if the argument is of the
form <code class="inline"><span class="q">"U+..."</span></code>
. Your best bet there for runtime Unicode by character
name is probably:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/use.html">use</a> <span class="w">charnames</span> <span class="s">(</span><span class="s">)</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$hebrew_alef_from_name</span></li><li> = <a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">"U"</span><span class="cm">,</span> <span class="i">charnames::vianame</span><span class="s">(</span><span class="q">"HEBREW LETTER ALEF"</span><span class="s">)</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><a name="Handling-Unicode"></a><h2>Handling Unicode</h2>
<p>Handling Unicode is for the most part transparent: just use the
strings as usual. Functions like <code class="inline"><a class="l_k" href="functions/index.html">index()</a></code>, <code class="inline"><a class="l_k" href="functions/length.html">length()</a></code>, and
<code class="inline"><a class="l_k" href="functions/substr.html">substr()</a></code> will work on the Unicode characters; regular expressions
will work on the Unicode characters (see <a href="perlunicode.html">perlunicode</a> and <a href="perlretut.html">perlretut</a>).</p>
<p>Note that Perl considers grapheme clusters to be separate characters, so for
example</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/print.html">print</a> <a class="l_k" href="functions/length.html">length</a><span class="s">(</span><span class="q">"\N{LATIN CAPITAL LETTER A}\N{COMBINING ACUTE ACCENT}"</span><span class="s">)</span><span class="cm">,</span></li><li> <span class="q">"\n"</span><span class="sc">;</span></li></ol></pre><p>will print 2, not 1. The only exception is that regular expressions
have <code class="inline">\<span class="w">X</span></code>
for matching an extended grapheme cluster. (Thus <code class="inline">\<span class="w">X</span></code>
in a
regular expression would match the entire sequence of both the example
characters.)</p>
<p>Life is not quite so transparent, however, when working with legacy
encodings, I/O, and certain special cases:</p>
<a name="Legacy-Encodings"></a><h2>Legacy Encodings</h2>
<p>When you combine legacy data and Unicode, the legacy data needs
to be upgraded to Unicode. Normally the legacy data is assumed to be
ISO 8859-1 (or EBCDIC, if applicable).</p>
<p>The <code class="inline"><span class="w">Encode</span></code>
module knows about many encodings and has interfaces
for doing conversions between those encodings:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/use.html">use</a> <span class="w">Encode</span> <span class="q">'decode'</span><span class="sc">;</span></li><li> <span class="i">$data</span> = <span class="i">decode</span><span class="s">(</span><span class="q">"iso-8859-3"</span><span class="cm">,</span> <span class="i">$data</span><span class="s">)</span><span class="sc">;</span> <span class="c"># convert from legacy to utf-8</span></li></ol></pre><a name="Unicode-I%2fO"></a><h2>Unicode I/O</h2>
<p>Normally, writing out Unicode data</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/print.html">print</a> <span class="i">FH</span> <span class="i">$some_string_with_unicode</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span></li></ol></pre><p>produces raw bytes that Perl happens to use to internally encode the
Unicode string. Perl's internal encoding depends on the system as
well as what characters happen to be in the string at the time. If
any of the characters are at code points <code class="inline"><span class="n">0x100</span></code>
or above, you will get
a warning. To ensure that the output is explicitly rendered in the
encoding you desire--and to avoid the warning--open the stream with
the desired encoding. Some examples:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/open.html">open</a> <span class="w">FH</span><span class="cm">,</span> <span class="q">">:utf8"</span><span class="cm">,</span> <span class="q">"file"</span><span class="sc">;</span></li><li></li><li> <a class="l_k" href="functions/open.html">open</a> <span class="w">FH</span><span class="cm">,</span> <span class="q">">:encoding(ucs2)"</span><span class="cm">,</span> <span class="q">"file"</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/open.html">open</a> <span class="w">FH</span><span class="cm">,</span> <span class="q">">:encoding(UTF-8)"</span><span class="cm">,</span> <span class="q">"file"</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/open.html">open</a> <span class="w">FH</span><span class="cm">,</span> <span class="q">">:encoding(shift_jis)"</span><span class="cm">,</span> <span class="q">"file"</span><span class="sc">;</span></li></ol></pre><p>and on already open streams, use <code class="inline"><a class="l_k" href="functions/binmode.html">binmode()</a></code>:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/binmode.html">binmode</a><span class="s">(</span><span class="w">STDOUT</span><span class="cm">,</span> <span class="q">":utf8"</span><span class="s">)</span><span class="sc">;</span></li><li></li><li> <a class="l_k" href="functions/binmode.html">binmode</a><span class="s">(</span><span class="w">STDOUT</span><span class="cm">,</span> <span class="q">":encoding(ucs2)"</span><span class="s">)</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/binmode.html">binmode</a><span class="s">(</span><span class="w">STDOUT</span><span class="cm">,</span> <span class="q">":encoding(UTF-8)"</span><span class="s">)</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/binmode.html">binmode</a><span class="s">(</span><span class="w">STDOUT</span><span class="cm">,</span> <span class="q">":encoding(shift_jis)"</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>The matching of encoding names is loose: case does not matter, and
many encodings have several aliases. Note that the <code class="inline"><span class="j">:</span><span class="w">utf8</span></code>
layer
must always be specified exactly like that; it is <i>not</i> subject to
the loose matching of encoding names. Also note that currently <code class="inline"><span class="j">:</span><span class="w">utf8</span></code>
is unsafe for
input, because it accepts the data without validating that it is indeed valid
UTF-8; you should instead use <code class="inline"><span class="j">:</span><span class="i">encoding</span><span class="s">(</span><span class="w">utf</span>-<span class="n">8</span><span class="s">)</span></code>
(with or without a
hyphen).</p>
<p>See <a href="PerlIO.html">PerlIO</a> for the <code class="inline"><span class="j">:</span><span class="w">utf8</span></code>
layer, <a href="PerlIO/encoding.html">PerlIO::encoding</a> and
<a href="Encode/PerlIO.html">Encode::PerlIO</a> for the <code class="inline"><span class="j">:</span><span class="i">encoding</span><span class="s">(</span><span class="s">)</span></code>
layer, and
<a href="Encode/Supported.html">Encode::Supported</a> for many encodings supported by the <code class="inline"><span class="w">Encode</span></code>
module.</p>
<p>Reading in a file that you know happens to be encoded in one of the
Unicode or legacy encodings does not magically turn the data into
Unicode in Perl's eyes. To do that, specify the appropriate
layer when opening files</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/open.html">open</a><span class="s">(</span><a class="l_k" href="functions/my.html">my</a> <span class="i">$fh</span><span class="cm">,</span><span class="q">'<:encoding(utf8)'</span><span class="cm">,</span> <span class="q">'anything'</span><span class="s">)</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$line_of_unicode</span> = <span class="q"><$fh></span><span class="sc">;</span></li><li></li><li> <a class="l_k" href="functions/open.html">open</a><span class="s">(</span><a class="l_k" href="functions/my.html">my</a> <span class="i">$fh</span><span class="cm">,</span><span class="q">'<:encoding(Big5)'</span><span class="cm">,</span> <span class="q">'anything'</span><span class="s">)</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$line_of_unicode</span> = <span class="q"><$fh></span><span class="sc">;</span></li></ol></pre><p>The I/O layers can also be specified more flexibly with
the <code class="inline"><a class="l_k" href="functions/open.html">open</a></code> pragma. See <a href="open.html">open</a>, or look at the following example.</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/use.html">use</a> <a class="l_k" href="functions/open.html">open</a> <span class="q">':encoding(utf8)'</span><span class="sc">;</span> <span class="c"># input/output default encoding will be</span></li><li> <span class="c"># UTF-8</span></li><li> <a class="l_k" href="functions/open.html">open</a> <span class="w">X</span><span class="cm">,</span> <span class="q">">file"</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/print.html">print</a> <span class="i">X</span> <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="n">0x100</span><span class="s">)</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/close.html">close</a> <span class="w">X</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/open.html">open</a> <span class="w">Y</span><span class="cm">,</span> <span class="q">"<file"</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/printf.html">printf</a> <span class="q">"%#x\n"</span><span class="cm">,</span> <a class="l_k" href="functions/ord.html">ord</a><span class="s">(</span><span class="q"><Y></span><span class="s">)</span><span class="sc">;</span> <span class="c"># this should print 0x100</span></li><li> <a class="l_k" href="functions/close.html">close</a> <span class="w">Y</span><span class="sc">;</span></li></ol></pre><p>With the <code class="inline"><a class="l_k" href="functions/open.html">open</a></code> pragma you can use the <code class="inline"><span class="j">:</span><span class="w">locale</span></code>
layer</p>
<pre class="verbatim"><ol><li> BEGIN <span class="s">{</span> <span class="i">$ENV</span>{<span class="w">LC_ALL</span>} = <span class="i">$ENV</span>{<span class="w">LANG</span>} = <span class="q">'ru_RU.KOI8-R'</span> <span class="s">}</span></li><li> <span class="c"># the :locale will probe the locale environment variables like</span></li><li> <span class="c"># LC_ALL</span></li><li> <a class="l_k" href="functions/use.html">use</a> <a class="l_k" href="functions/open.html">open</a> <span class="w">OUT</span> <span class="cm">=></span> <span class="q">':locale'</span><span class="sc">;</span> <span class="c"># russki parusski</span></li><li> <a class="l_k" href="functions/open.html">open</a><span class="s">(</span><span class="w">O</span><span class="cm">,</span> <span class="q">">koi8"</span><span class="s">)</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/print.html">print</a> <span class="i">O</span> <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="n">0x430</span><span class="s">)</span><span class="sc">;</span> <span class="c"># Unicode CYRILLIC SMALL LETTER A = KOI8-R 0xc1</span></li><li> <a class="l_k" href="functions/close.html">close</a> <span class="w">O</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/open.html">open</a><span class="s">(</span><span class="w">I</span><span class="cm">,</span> <span class="q">"<koi8"</span><span class="s">)</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/printf.html">printf</a> <span class="q">"%#x\n"</span><span class="cm">,</span> <a class="l_k" href="functions/ord.html">ord</a><span class="s">(</span><span class="q"><I></span><span class="s">)</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span> <span class="c"># this should print 0xc1</span></li><li> <a class="l_k" href="functions/close.html">close</a> <span class="w">I</span><span class="sc">;</span></li></ol></pre><p>These methods install a transparent filter on the I/O stream that
converts data from the specified encoding when it is read in from the
stream. The result is always Unicode.</p>
<p>The <a href="open.html">open</a> pragma affects all the <code class="inline"><a class="l_k" href="functions/open.html">open()</a></code> calls after the pragma by
setting default layers. If you want to affect only certain
streams, use explicit layers directly in the <code class="inline"><a class="l_k" href="functions/open.html">open()</a></code> call.</p>
<p>You can switch encodings on an already opened stream by using
<code class="inline"><a class="l_k" href="functions/binmode.html">binmode()</a></code>; see <a href="functions/binmode.html">binmode</a>.</p>
<p>The <code class="inline"><span class="j">:</span><span class="w">locale</span></code>
does not currently work with
<code class="inline"><a class="l_k" href="functions/open.html">open()</a></code> and <code class="inline"><a class="l_k" href="functions/binmode.html">binmode()</a></code>, only with the <code class="inline"><a class="l_k" href="functions/open.html">open</a></code> pragma. The
<code class="inline"><span class="j">:</span><span class="w">utf8</span></code>
and <code class="inline"><span class="j">:</span><span class="i">encoding</span><span class="s">(</span>...<span class="s">)</span></code>
methods do work with all of <code class="inline"><a class="l_k" href="functions/open.html">open()</a></code>,
<code class="inline"><a class="l_k" href="functions/binmode.html">binmode()</a></code>, and the <code class="inline"><a class="l_k" href="functions/open.html">open</a></code> pragma.</p>
<p>Similarly, you may use these I/O layers on output streams to
automatically convert Unicode to the specified encoding when it is
written to the stream. For example, the following snippet copies the
contents of the file "text.jis" (encoded as ISO-2022-JP, aka JIS) to
the file "text.utf8", encoded as UTF-8:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/open.html">open</a><span class="s">(</span><a class="l_k" href="functions/my.html">my</a> <span class="i">$nihongo</span><span class="cm">,</span> <span class="q">'<:encoding(iso-2022-jp)'</span><span class="cm">,</span> <span class="q">'text.jis'</span><span class="s">)</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/open.html">open</a><span class="s">(</span><a class="l_k" href="functions/my.html">my</a> <span class="i">$unicode</span><span class="cm">,</span> <span class="q">'>:utf8'</span><span class="cm">,</span> <span class="q">'text.utf8'</span><span class="s">)</span><span class="sc">;</span></li><li> while <span class="s">(</span><span class="q"><$nihongo></span><span class="s">)</span> <span class="s">{</span> <a class="l_k" href="functions/print.html">print</a> <span class="i">$unicode</span> <span class="i">$_</span> <span class="s">}</span></li></ol></pre><p>The naming of encodings, both by the <code class="inline"><a class="l_k" href="functions/open.html">open()</a></code> and by the <code class="inline"><a class="l_k" href="functions/open.html">open</a></code>
pragma allows for flexible names: <code class="inline"><span class="w">koi8</span>-r</code>
and <code class="inline"><span class="w">KOI8R</span></code>
will both be
understood.</p>
<p>Common encodings recognized by ISO, MIME, IANA, and various other
standardisation organisations are recognised; for a more detailed
list see <a href="Encode/Supported.html">Encode::Supported</a>.</p>
<p><code class="inline"><a class="l_k" href="functions/read.html">read()</a></code> reads characters and returns the number of characters.
<code class="inline"><a class="l_k" href="functions/seek.html">seek()</a></code> and <code class="inline"><a class="l_k" href="functions/tell.html">tell()</a></code> operate on byte counts, as do <code class="inline"><a class="l_k" href="functions/sysread.html">sysread()</a></code>
and <code class="inline"><a class="l_k" href="functions/sysseek.html">sysseek()</a></code>.</p>
<p>Notice that because of the default behaviour of not doing any
conversion upon input if there is no default layer,
it is easy to mistakenly write code that keeps on expanding a file
by repeatedly encoding the data:</p>
<pre class="verbatim"><ol><li> <span class="c"># BAD CODE WARNING</span></li><li> <a class="l_k" href="functions/open.html">open</a> <span class="w">F</span><span class="cm">,</span> <span class="q">"file"</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/local.html">local</a> <span class="i">$/</span><span class="sc">;</span> <span class="c">## read in the whole file of 8-bit characters</span></li><li> <span class="i">$t</span> = <span class="q"><F></span><span class="sc">;</span></li><li> <a class="l_k" href="functions/close.html">close</a> <span class="w">F</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/open.html">open</a> <span class="w">F</span><span class="cm">,</span> <span class="q">">:encoding(utf8)"</span><span class="cm">,</span> <span class="q">"file"</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/print.html">print</a> <span class="i">F</span> <span class="i">$t</span><span class="sc">;</span> <span class="c">## convert to UTF-8 on output</span></li><li> <a class="l_k" href="functions/close.html">close</a> <span class="w">F</span><span class="sc">;</span></li></ol></pre><p>If you run this code twice, the contents of the <i>file</i> will be twice
UTF-8 encoded. A <code class="inline"><a class="l_k" href="functions/use.html">use</a> <a class="l_k" href="functions/open.html">open</a> <span class="q">':encoding(utf8)'</span></code>
would have avoided the
bug, or explicitly opening also the <i>file</i> for input as UTF-8.</p>
<p><b>NOTE</b>: the <code class="inline"><span class="j">:</span><span class="w">utf8</span></code>
and <code class="inline"><span class="j">:</span><span class="w">encoding</span></code>
features work only if your
Perl has been built with <a href="PerlIO.html">PerlIO</a>, which is the default
on most systems.</p>
<a name="Displaying-Unicode-As-Text"></a><h2>Displaying Unicode As Text</h2>
<p>Sometimes you might want to display Perl scalars containing Unicode as
simple ASCII (or EBCDIC) text. The following subroutine converts
its argument so that Unicode characters with code points greater than
255 are displayed as <code class="inline">\<span class="i">x</span><span class="s">{</span>...<span class="s">}</span></code>
, control characters (like <code class="inline">\<span class="w">n</span></code>
) are
displayed as <code class="inline">\<span class="w">x</span>..</code>
, and the rest of the characters as themselves:</p>
<pre class="verbatim"><ol><li><a name="nice_string"></a> sub <span class="m">nice_string</span> <span class="s">{</span></li><li> <a class="l_k" href="functions/join.html">join</a><span class="s">(</span><span class="q">""</span><span class="cm">,</span></li><li> <a class="l_k" href="functions/map.html">map</a> <span class="s">{</span> <span class="i">$_</span> > <span class="n">255</span> <span class="c"># if wide character...</span></li><li> ? <a class="l_k" href="functions/sprintf.html">sprintf</a><span class="s">(</span><span class="q">"\\x{%04X}"</span><span class="cm">,</span> <span class="i">$_</span><span class="s">)</span> <span class="c"># \x{...}</span></li><li> <span class="co">:</span> <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="i">$_</span><span class="s">)</span> =~ <span class="q">/[[:cntrl:]]/</span> <span class="c"># else if control character...</span></li><li> ? <a class="l_k" href="functions/sprintf.html">sprintf</a><span class="s">(</span><span class="q">"\\x%02X"</span><span class="cm">,</span> <span class="i">$_</span><span class="s">)</span> <span class="c"># \x..</span></li><li> <span class="co">:</span> <a class="l_k" href="functions/quotemeta.html">quotemeta</a><span class="s">(</span><a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="i">$_</span><span class="s">)</span><span class="s">)</span> <span class="c"># else quoted or as themselves</span></li><li> <span class="s">}</span> <a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">"W*"</span><span class="cm">,</span> <span class="i">$_</span>[<span class="n">0</span>]<span class="s">)</span><span class="s">)</span><span class="sc">;</span> <span class="c"># unpack Unicode characters</span></li><li> <span class="s">}</span></li></ol></pre><p>For example,</p>
<pre class="verbatim"><ol><li> <span class="i">nice_string</span><span class="s">(</span><span class="q">"foo\x{100}bar\n"</span><span class="s">)</span></li></ol></pre><p>returns the string</p>
<pre class="verbatim"><ol><li> <span class="q">'foo\x{0100}bar\x0A'</span></li></ol></pre><p>which is ready to be printed.</p>
<p>(<code class="inline">\\<span class="w">x</span><span class="s">{</span><span class="s">}</span></code>
is used here instead of <code class="inline">\\<span class="w">N</span><span class="s">{</span><span class="s">}</span></code>
, since it's most likely that
you want to see what the native values are.)</p>
<a name="Special-Cases"></a><h2>Special Cases</h2>
<ul>
<li>
<p>Bit Complement Operator ~ And vec()</p>
<p>The bit complement operator <code class="inline">~</code>
may produce surprising results if
used on strings containing characters with ordinal values above
255. In such a case, the results are consistent with the internal
encoding of the characters, but not with much else. So don't do
that. Similarly for <code class="inline"><a class="l_k" href="functions/vec.html">vec()</a></code>: you will be operating on the
internally-encoded bit patterns of the Unicode characters, not on
the code point values, which is very probably not what you want.</p>
</li>
<li>
<p>Peeking At Perl's Internal Encoding</p>
<p>Normal users of Perl should never care how Perl encodes any particular
Unicode string (because the normal ways to get at the contents of a
string with Unicode--via input and output--should always be via
explicitly-defined I/O layers). But if you must, there are two
ways of looking behind the scenes.</p>
<p>One way of peeking inside the internal encoding of Unicode characters
is to use <code class="inline">unpack("C*", ...</code> to get the bytes of whatever the string
encoding happens to be, or <code class="inline"><a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">"U0.."</span><span class="cm">,</span> ...<span class="s">)</span></code>
to get the bytes of the
UTF-8 encoding:</p>
<pre class="verbatim"><ol><li> <span class="c"># this prints c4 80 for the UTF-8 bytes 0xc4 0x80</span></li><li> <a class="l_k" href="functions/print.html">print</a> <a class="l_k" href="functions/join.html">join</a><span class="s">(</span><span class="q">" "</span><span class="cm">,</span> <a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">"U0(H2)*"</span><span class="cm">,</span> <a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">"U"</span><span class="cm">,</span> <span class="n">0x100</span><span class="s">)</span><span class="s">)</span><span class="s">)</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span></li></ol></pre><p>Yet another way would be to use the Devel::Peek module:</p>
<pre class="verbatim"><ol><li> <span class="w">perl</span> -<span class="w">MDevel::Peek</span> -e <span class="q">'Dump(chr(0x100))'</span></li></ol></pre><p>That shows the <code class="inline"><span class="w">UTF8</span></code>
flag in FLAGS and both the UTF-8 bytes
and Unicode characters in <code class="inline"><span class="w">PV</span></code>
. See also later in this document
the discussion about the <code class="inline"><span class="i">utf8::is_utf8</span><span class="s">(</span><span class="s">)</span></code>
function.</p>
</li>
</ul>
<a name="Advanced-Topics"></a><h2>Advanced Topics</h2>
<ul>
<li>
<p>String Equivalence</p>
<p>The question of string equivalence turns somewhat complicated
in Unicode: what do you mean by "equal"?</p>
<p>(Is <code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span> <span class="w">WITH</span> <span class="w">ACUTE</span></code>
equal to
<code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span></code>
?)</p>
<p>The short answer is that by default Perl compares equivalence (<code class="inline">eq</code>
,
<code class="inline">ne</code>
) based only on code points of the characters. In the above
case, the answer is no (because 0x00C1 != 0x0041). But sometimes, any
CAPITAL LETTER A's should be considered equal, or even A's of any case.</p>
<p>The long answer is that you need to consider character normalization
and casing issues: see <a href="Unicode/Normalize.html">Unicode::Normalize</a>, Unicode Technical Report #15,
<a href="http://search.cpan.org/perldoc/http:#%2fwww.unicode.org%2funicode%2freports%2ftr15">Unicode Normalization Forms</a> and
sections on case mapping in the <a href="http://search.cpan.org/perldoc/http:#%2fwww.unicode.org">Unicode Standard</a>.</p>
<p>As of Perl 5.8.0, the "Full" case-folding of <i>Case
Mappings/SpecialCasing</i> is implemented, but bugs remain in <code class="inline"><a class="l_k" href="functions/qr.html">qr//i</a></code> with them,
mostly fixed by 5.14, and essentially entirely by 5.18.</p>
</li>
<li>
<p>String Collation</p>
<p>People like to see their strings nicely sorted--or as Unicode
parlance goes, collated. But again, what do you mean by collate?</p>
<p>(Does <code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span> <span class="w">WITH</span> <span class="w">ACUTE</span></code>
come before or after
<code class="inline"><span class="w">LATIN</span> <span class="w">CAPITAL</span> <span class="w">LETTER</span> <span class="w">A</span> <span class="w">WITH</span> <span class="w">GRAVE</span></code>
?)</p>
<p>The short answer is that by default, Perl compares strings (<code class="inline">lt</code>
,
<code class="inline">le</code>
, <code class="inline">cmp</code>
, <code class="inline">ge</code>
, <code class="inline">gt</code>
) based only on the code points of the
characters. In the above case, the answer is "after", since
<code class="inline"><span class="n">0x00C1</span></code>
> <code class="inline"><span class="n">0x00C0</span></code>
.</p>
<p>The long answer is that "it depends", and a good answer cannot be
given without knowing (at the very least) the language context.
See <a href="Unicode/Collate.html">Unicode::Collate</a>, and <i>Unicode Collation Algorithm</i>
<a href="http://www.unicode.org/unicode/reports/tr10/">http://www.unicode.org/unicode/reports/tr10/</a></p>
</li>
</ul>
<a name="Miscellaneous"></a><h2>Miscellaneous</h2>
<ul>
<li>
<p>Character Ranges and Classes</p>
<p>Character ranges in regular expression bracketed character classes ( e.g.,
<code class="inline"><span class="q">/[a-z]/</span></code>
) and in the <code class="inline"><a class="l_k" href="functions/tr.html">tr///</a></code> (also known as <code class="inline"><a class="l_k" href="functions/y.html">y///</a></code>) operator are not
magically Unicode-aware. What this means is that <code class="inline"><span class="s">[</span><span class="w">A</span>-<span class="w">Za</span>-z<span class="s">]</span></code>
will not
magically start to mean "all alphabetic letters" (not that it does mean that
even for 8-bit characters; for those, if you are using locales (<a href="perllocale.html">perllocale</a>),
use <code class="inline"><span class="q">/[[:alpha:]]/</span></code>
; and if not, use the 8-bit-aware property <code class="inline">\<span class="i">p</span><span class="s">{</span><span class="w">alpha</span><span class="s">}</span></code>
).</p>
<p>All the properties that begin with <code class="inline">\<span class="w">p</span></code>
(and its inverse <code class="inline">\<span class="w">P</span></code>
) are actually
character classes that are Unicode-aware. There are dozens of them, see
<a href="perluniprops.html">perluniprops</a>.</p>
<p>Starting in v5.22, you can use Unicode code points as the end points of
regular expression pattern character ranges, and the range will include
all Unicode code points that lie between those end points, inclusive.</p>
<pre class="verbatim"><ol><li> <span class="q">qr/ [\N{U+03]-\N{U+20}] /x</span></li></ol></pre><p>includes the code points
<code class="inline">\<span class="i">N</span><span class="s">{</span><span class="w">U</span>+<span class="n">03</span><span class="s">}</span></code>
, <code class="inline">\<span class="i">N</span><span class="s">{</span><span class="w">U</span>+<span class="n">04</span><span class="s">}</span></code>
, ..., <code class="inline">\<span class="i">N</span><span class="s">{</span><span class="w">U</span>+<span class="n">20</span><span class="s">}</span></code>
.</p>
<p>(It is planned to extend this behavior to ranges in <code class="inline"><a class="l_k" href="functions/tr.html">tr///</a></code> in Perl
v5.24.)</p>
</li>
<li>
<p>String-To-Number Conversions</p>
<p>Unicode does define several other decimal--and numeric--characters
besides the familiar 0 to 9, such as the Arabic and Indic digits.
Perl does not support string-to-number conversion for digits other
than ASCII <code class="inline"><span class="n">0</span></code>
to <code class="inline"><span class="n">9</span></code>
(and ASCII <code class="inline"><span class="w">a</span></code>
to <code class="inline"><span class="w">f</span></code>
for hexadecimal).
To get safe conversions from any Unicode string, use
<a href="Unicode/UCD.html#num()">num() in Unicode::UCD</a>.</p>
</li>
</ul>
<a name="Questions-With-Answers"></a><h2>Questions With Answers</h2>
<ul>
<li>
<p>Will My Old Scripts Break?</p>
<p>Very probably not. Unless you are generating Unicode characters
somehow, old behaviour should be preserved. About the only behaviour
that has changed and which could start generating Unicode is the old
behaviour of <code class="inline"><a class="l_k" href="functions/chr.html">chr()</a></code> where supplying an argument more than 255
produced a character modulo 255. <code class="inline"><a class="l_k" href="functions/chr.html">chr(300)</a></code>, for example, was equal
to <code class="inline"><a class="l_k" href="functions/chr.html">chr(45)</a></code> or "-" (in ASCII), now it is LATIN CAPITAL LETTER I WITH
BREVE.</p>
</li>
<li>
<p>How Do I Make My Scripts Work With Unicode?</p>
<p>Very little work should be needed since nothing changes until you
generate Unicode data. The most important thing is getting input as
Unicode; for that, see the earlier I/O discussion.
To get full seamless Unicode support, add
<code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">feature</span> <span class="q">'unicode_strings'</span></code>
(or <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="n">5.012</span></code>
or higher) to your
script.</p>
</li>
<li>
<p>How Do I Know Whether My String Is In Unicode?</p>
<p>You shouldn't have to care. But you may if your Perl is before 5.14.0
or you haven't specified <code class="inline"><a class="l_k" href="functions/use.html">use</a> <span class="w">feature</span> <span class="q">'unicode_strings'</span></code>
or <code class="inline"><a class="l_k" href="functions/use.html">use</a>
<span class="n">5.012</span></code>
(or higher) because otherwise the rules for the code points
in the range 128 to 255 are different depending on
whether the string they are contained within is in Unicode or not.
(See <a href="perlunicode.html#When-Unicode-Does-Not-Happen">When Unicode Does Not Happen in perlunicode</a>.)</p>
<p>To determine if a string is in Unicode, use:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/print.html">print</a> <span class="i">utf8::is_utf8</span><span class="s">(</span><span class="i">$string</span><span class="s">)</span> ? <span class="n">1</span> <span class="co">:</span> <span class="n">0</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span></li></ol></pre><p>But note that this doesn't mean that any of the characters in the
string are necessary UTF-8 encoded, or that any of the characters have
code points greater than 0xFF (255) or even 0x80 (128), or that the
string has any characters at all. All the <code class="inline"><span class="i">is_utf8</span><span class="s">(</span><span class="s">)</span></code>
does is to
return the value of the internal "utf8ness" flag attached to the
<code class="inline"><span class="i">$string</span></code>
. If the flag is off, the bytes in the scalar are interpreted
as a single byte encoding. If the flag is on, the bytes in the scalar
are interpreted as the (variable-length, potentially multi-byte) UTF-8 encoded
code points of the characters. Bytes added to a UTF-8 encoded string are
automatically upgraded to UTF-8. If mixed non-UTF-8 and UTF-8 scalars
are merged (double-quoted interpolation, explicit concatenation, or
printf/sprintf parameter substitution), the result will be UTF-8 encoded
as if copies of the byte strings were upgraded to UTF-8: for example,</p>
<pre class="verbatim"><ol><li> <span class="i">$a</span> = <span class="q">"ab\x80c"</span><span class="sc">;</span></li><li> <span class="i">$b</span> = <span class="q">"\x{100}"</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/print.html">print</a> <span class="q">"$a = $b\n"</span><span class="sc">;</span></li></ol></pre><p>the output string will be UTF-8-encoded <code class="inline"><span class="w">ab</span>\<span class="w">x80c</span> = \<span class="i">x</span><span class="s">{</span><span class="n">100</span><span class="s">}</span>\<span class="w">n</span></code>
, but
<code class="inline"><span class="i">$a</span></code>
will stay byte-encoded.</p>
<p>Sometimes you might really need to know the byte length of a string
instead of the character length. For that use either the
<code class="inline"><span class="i">Encode::encode_utf8</span><span class="s">(</span><span class="s">)</span></code>
function or the <code class="inline"><span class="w">bytes</span></code>
pragma
and the <code class="inline"><a class="l_k" href="functions/length.html">length()</a></code> function:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$unicode</span> = <a class="l_k" href="functions/chr.html">chr</a><span class="s">(</span><span class="n">0x100</span><span class="s">)</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/print.html">print</a> <a class="l_k" href="functions/length.html">length</a><span class="s">(</span><span class="i">$unicode</span><span class="s">)</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span> <span class="c"># will print 1</span></li><li> <a class="l_k" href="functions/require.html">require</a> <span class="w">Encode</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/print.html">print</a> <a class="l_k" href="functions/length.html">length</a><span class="s">(</span><span class="i">Encode::encode_utf8</span><span class="s">(</span><span class="i">$unicode</span><span class="s">)</span><span class="s">)</span><span class="cm">,</span><span class="q">"\n"</span><span class="sc">;</span> <span class="c"># will print 2</span></li><li> <a class="l_k" href="functions/use.html">use</a> <span class="w">bytes</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/print.html">print</a> <a class="l_k" href="functions/length.html">length</a><span class="s">(</span><span class="i">$unicode</span><span class="s">)</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span> <span class="c"># will also print 2</span></li><li> <span class="c"># (the 0xC4 0x80 of the UTF-8)</span></li><li> <a class="l_k" href="functions/no.html">no</a> <span class="w">bytes</span><span class="sc">;</span></li></ol></pre></li>
<li>
<p>How Do I Find Out What Encoding a File Has?</p>
<p>You might try <a href="Encode/Guess.html">Encode::Guess</a>, but it has a number of limitations.</p>
</li>
<li>
<p>How Do I Detect Data That's Not Valid In a Particular Encoding?</p>
<p>Use the <code class="inline"><span class="w">Encode</span></code>
package to try converting it.
For example,</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/use.html">use</a> <span class="w">Encode</span> <span class="q">'decode_utf8'</span><span class="sc">;</span></li><li></li><li> if <span class="s">(</span><a class="l_k" href="functions/eval.html">eval</a> <span class="s">{</span> <span class="i">decode_utf8</span><span class="s">(</span><span class="i">$string</span><span class="cm">,</span> <span class="w">Encode::FB_CROAK</span><span class="s">)</span><span class="sc">;</span> <span class="n">1</span> <span class="s">}</span><span class="s">)</span> <span class="s">{</span></li><li> <span class="c"># $string is valid utf8</span></li><li> <span class="s">}</span> else <span class="s">{</span></li><li> <span class="c"># $string is not valid utf8</span></li><li> <span class="s">}</span></li></ol></pre><p>Or use <code class="inline"><a class="l_k" href="functions/unpack.html">unpack</a></code> to try decoding it:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/use.html">use</a> <span class="w">warnings</span><span class="sc">;</span></li><li> <span class="i">@chars</span> = <a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">"C0U*"</span><span class="cm">,</span> <span class="i">$string_of_bytes_that_I_think_is_utf8</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>If invalid, a <code class="inline">Malformed UTF-8 character</code> warning is produced. The "C0" means
"process the string character per character". Without that, the
<code class="inline"><a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">"U*"</span><span class="cm">,</span> ...<span class="s">)</span></code>
would work in <code class="inline"><span class="w">U0</span></code>
mode (the default if the format
string starts with <code class="inline"><span class="w">U</span></code>
) and it would return the bytes making up the UTF-8
encoding of the target string, something that will always work.</p>
</li>
<li>
<p>How Do I Convert Binary Data Into a Particular Encoding, Or Vice Versa?</p>
<p>This probably isn't as useful as you might think.
Normally, you shouldn't need to.</p>
<p>In one sense, what you are asking doesn't make much sense: encodings
are for characters, and binary data are not "characters", so converting
"data" into some encoding isn't meaningful unless you know in what
character set and encoding the binary data is in, in which case it's
not just binary data, now is it?</p>
<p>If you have a raw sequence of bytes that you know should be
interpreted via a particular encoding, you can use <code class="inline"><span class="w">Encode</span></code>
:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/use.html">use</a> <span class="w">Encode</span> <span class="q">'from_to'</span><span class="sc">;</span></li><li> <span class="i">from_to</span><span class="s">(</span><span class="i">$data</span><span class="cm">,</span> <span class="q">"iso-8859-1"</span><span class="cm">,</span> <span class="q">"utf-8"</span><span class="s">)</span><span class="sc">;</span> <span class="c"># from latin-1 to utf-8</span></li></ol></pre><p>The call to <code class="inline"><span class="i">from_to</span><span class="s">(</span><span class="s">)</span></code>
changes the bytes in <code class="inline"><span class="i">$data</span></code>
, but nothing
material about the nature of the string has changed as far as Perl is
concerned. Both before and after the call, the string <code class="inline"><span class="i">$data</span></code>
contains just a bunch of 8-bit bytes. As far as Perl is concerned,
the encoding of the string remains as "system-native 8-bit bytes".</p>
<p>You might relate this to a fictional 'Translate' module:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/use.html">use</a> <span class="w">Translate</span><span class="sc">;</span></li><li> <a class="l_k" href="functions/my.html">my</a> <span class="i">$phrase</span> = <span class="q">"Yes"</span><span class="sc">;</span></li><li> <span class="i">Translate::from_to</span><span class="s">(</span><span class="i">$phrase</span><span class="cm">,</span> <span class="q">'english'</span><span class="cm">,</span> <span class="q">'deutsch'</span><span class="s">)</span><span class="sc">;</span></li><li> <span class="c">## phrase now contains "Ja"</span></li></ol></pre><p>The contents of the string changes, but not the nature of the string.
Perl doesn't know any more after the call than before that the
contents of the string indicates the affirmative.</p>
<p>Back to converting data. If you have (or want) data in your system's
native 8-bit encoding (e.g. Latin-1, EBCDIC, etc.), you can use
pack/unpack to convert to/from Unicode.</p>
<pre class="verbatim"><ol><li> <span class="i">$native_string</span> = <a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">"W*"</span><span class="cm">,</span> <a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">"U*"</span><span class="cm">,</span> <span class="i">$Unicode_string</span><span class="s">)</span><span class="s">)</span><span class="sc">;</span></li><li> <span class="i">$Unicode_string</span> = <a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">"U*"</span><span class="cm">,</span> <a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">"W*"</span><span class="cm">,</span> <span class="i">$native_string</span><span class="s">)</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>If you have a sequence of bytes you <b>know</b> is valid UTF-8,
but Perl doesn't know it yet, you can make Perl a believer, too:</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/use.html">use</a> <span class="w">Encode</span> <span class="q">'decode_utf8'</span><span class="sc">;</span></li><li> <span class="i">$Unicode</span> = <span class="i">decode_utf8</span><span class="s">(</span><span class="i">$bytes</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>or:</p>
<pre class="verbatim"><ol><li> <span class="i">$Unicode</span> = <a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">"U0a*"</span><span class="cm">,</span> <span class="i">$bytes</span><span class="s">)</span><span class="sc">;</span></li></ol></pre><p>You can find the bytes that make up a UTF-8 sequence with</p>
<pre class="verbatim"><ol><li> <span class="i">@bytes</span> = <a class="l_k" href="functions/unpack.html">unpack</a><span class="s">(</span><span class="q">"C*"</span><span class="cm">,</span> <span class="i">$Unicode_string</span><span class="s">)</span></li></ol></pre><p>and you can create well-formed Unicode with</p>
<pre class="verbatim"><ol><li> <span class="i">$Unicode_string</span> = <a class="l_k" href="functions/pack.html">pack</a><span class="s">(</span><span class="q">"U*"</span><span class="cm">,</span> <span class="n">0xff</span><span class="cm">,</span> ...<span class="s">)</span></li></ol></pre></li>
<li>
<p>How Do I Display Unicode? How Do I Input Unicode?</p>
<p>See <a href="http://www.alanwood.net/unicode/">http://www.alanwood.net/unicode/</a> and
<a href="http://www.cl.cam.ac.uk/~mgk25/unicode.html">http://www.cl.cam.ac.uk/~mgk25/unicode.html</a></p>
</li>
<li>
<p>How Does Unicode Work With Traditional Locales?</p>
<p>If your locale is a UTF-8 locale, starting in Perl v5.20, Perl works
well for all categories except <code class="inline"><span class="w">LC_COLLATE</span></code>
dealing with sorting and
the <code class="inline">cmp</code>
operator.</p>
<p>For other locales, starting in Perl 5.16, you can specify</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/use.html">use</a> <span class="w">locale</span> <span class="q">':not_characters'</span><span class="sc">;</span></li></ol></pre><p>to get Perl to work well with them. The catch is that you
have to translate from the locale character set to/from Unicode
yourself. See <a href="#Unicode-I%2fO">Unicode I/O</a> above for how to</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/use.html">use</a> <a class="l_k" href="functions/open.html">open</a> <span class="q">':locale'</span><span class="sc">;</span></li></ol></pre><p>to accomplish this, but full details are in <a href="perllocale.html#Unicode-and-UTF-8">Unicode and UTF-8 in perllocale</a>, including gotchas that happen if you don't specify
<code class="inline"><span class="j">:</span><span class="w">not_characters</span></code>
.</p>
</li>
</ul>
<a name="Hexadecimal-Notation"></a><h2>Hexadecimal Notation</h2>
<p>The Unicode standard prefers using hexadecimal notation because
that more clearly shows the division of Unicode into blocks of 256 characters.
Hexadecimal is also simply shorter than decimal. You can use decimal
notation, too, but learning to use hexadecimal just makes life easier
with the Unicode standard. The <code class="inline"><span class="w">U</span>+<span class="w">HHHH</span></code>
notation uses hexadecimal,
for example.</p>
<p>The <code class="inline"><span class="n">0</span>x</code>
prefix means a hexadecimal number, the digits are 0-9 <i>and</i>
a-f (or A-F, case doesn't matter). Each hexadecimal digit represents
four bits, or half a byte. <code class="inline"><a class="l_k" href="functions/print.html">print</a> <span class="n">0</span>x...<span class="cm">,</span> <span class="q">"\n"</span></code>
will show a
hexadecimal number in decimal, and <code class="inline"><a class="l_k" href="functions/printf.html">printf</a> <span class="q">"%x\n"</span><span class="cm">,</span> <span class="i">$decimal</span></code>
will
show a decimal number in hexadecimal. If you have just the
"hex digits" of a hexadecimal number, you can use the <code class="inline"><a class="l_k" href="functions/hex.html">hex()</a></code> function.</p>
<pre class="verbatim"><ol><li> <a class="l_k" href="functions/print.html">print</a> <span class="n">0x0009</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span> <span class="c"># 9</span></li><li> <a class="l_k" href="functions/print.html">print</a> <span class="n">0x000a</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span> <span class="c"># 10</span></li><li> <a class="l_k" href="functions/print.html">print</a> <span class="n">0x000f</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span> <span class="c"># 15</span></li><li> <a class="l_k" href="functions/print.html">print</a> <span class="n">0x0010</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span> <span class="c"># 16</span></li><li> <a class="l_k" href="functions/print.html">print</a> <span class="n">0x0011</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span> <span class="c"># 17</span></li><li> <a class="l_k" href="functions/print.html">print</a> <span class="n">0x0100</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span> <span class="c"># 256</span></li><li></li><li> <a class="l_k" href="functions/print.html">print</a> <span class="n">0x0041</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span> <span class="c"># 65</span></li><li></li><li> <a class="l_k" href="functions/printf.html">printf</a> <span class="q">"%x\n"</span><span class="cm">,</span> <span class="n">65</span><span class="sc">;</span> <span class="c"># 41</span></li><li> <a class="l_k" href="functions/printf.html">printf</a> <span class="q">"%#x\n"</span><span class="cm">,</span> <span class="n">65</span><span class="sc">;</span> <span class="c"># 0x41</span></li><li></li><li> <a class="l_k" href="functions/print.html">print</a> <a class="l_k" href="functions/hex.html">hex</a><span class="s">(</span><span class="q">"41"</span><span class="s">)</span><span class="cm">,</span> <span class="q">"\n"</span><span class="sc">;</span> <span class="c"># 65</span></li></ol></pre><a name="Further-Resources"></a><h2>Further Resources</h2>
<ul>
<li>
<p>Unicode Consortium</p>
<p><a href="http://www.unicode.org/">http://www.unicode.org/</a></p>
</li>
<li>
<p>Unicode FAQ</p>
<p><a href="http://www.unicode.org/unicode/faq/">http://www.unicode.org/unicode/faq/</a></p>
</li>
<li>
<p>Unicode Glossary</p>
<p><a href="http://www.unicode.org/glossary/">http://www.unicode.org/glossary/</a></p>
</li>
<li>
<p>Unicode Recommended Reading List</p>
<p>The Unicode Consortium has a list of articles and books, some of which
give a much more in depth treatment of Unicode:
<a href="http://unicode.org/resources/readinglist.html">http://unicode.org/resources/readinglist.html</a></p>
</li>
<li>
<p>Unicode Useful Resources</p>
<p><a href="http://www.unicode.org/unicode/onlinedat/resources.html">http://www.unicode.org/unicode/onlinedat/resources.html</a></p>
</li>
<li>
<p>Unicode and Multilingual Support in HTML, Fonts, Web Browsers and Other Applications</p>
<p><a href="http://www.alanwood.net/unicode/">http://www.alanwood.net/unicode/</a></p>
</li>
<li>
<p>UTF-8 and Unicode FAQ for Unix/Linux</p>
<p><a href="http://www.cl.cam.ac.uk/~mgk25/unicode.html">http://www.cl.cam.ac.uk/~mgk25/unicode.html</a></p>
</li>
<li>
<p>Legacy Character Sets</p>
<p><a href="http://www.czyborra.com/">http://www.czyborra.com/</a>
<a href="http://www.eki.ee/letter/">http://www.eki.ee/letter/</a></p>
</li>
<li>
<p>You can explore various information from the Unicode data files using
the <code class="inline"><span class="w">Unicode::UCD</span></code>
module.</p>
</li>
</ul>
<a name="UNICODE-IN-OLDER-PERLS"></a><h1>UNICODE IN OLDER PERLS</h1>
<p>If you cannot upgrade your Perl to 5.8.0 or later, you can still
do some Unicode processing by using the modules <code class="inline"><span class="w">Unicode::String</span></code>
,
<code class="inline"><span class="w">Unicode::Map8</span></code>
, and <code class="inline"><span class="w">Unicode::Map</span></code>
, available from CPAN.
If you have the GNU recode installed, you can also use the
Perl front-end <code class="inline"><span class="w">Convert::Recode</span></code>
for character conversions.</p>
<p>The following are fast conversions from ISO 8859-1 (Latin-1) bytes
to UTF-8 bytes and back, the code works even with older Perl 5 versions.</p>
<pre class="verbatim"><ol><li> <span class="c"># ISO 8859-1 to UTF-8</span></li><li> <span class="q">s/([\x80-\xFF])/chr(0xC0|ord($1)>>6).chr(0x80|ord($1)&0x3F)/eg</span><span class="sc">;</span></li><li></li><li> <span class="c"># UTF-8 to ISO 8859-1</span></li><li> <span class="q">s/([\xC2\xC3])([\x80-\xBF])/chr(ord($1)<<6&0xC0|ord($2)&0x3F)/eg</span><span class="sc">;</span></li></ol></pre><a name="SEE-ALSO"></a><h1>SEE ALSO</h1>
<p><a href="perlunitut.html">perlunitut</a>, <a href="perlunicode.html">perlunicode</a>, <a href="Encode.html">Encode</a>, <a href="open.html">open</a>, <a href="utf8.html">utf8</a>, <a href="bytes.html">bytes</a>,
<a href="perlretut.html">perlretut</a>, <a href="perlrun.html">perlrun</a>, <a href="Unicode/Collate.html">Unicode::Collate</a>, <a href="Unicode/Normalize.html">Unicode::Normalize</a>,
<a href="Unicode/UCD.html">Unicode::UCD</a></p>
<a name="ACKNOWLEDGMENTS"></a><h1>ACKNOWLEDGMENTS</h1>
<p>Thanks to the kind readers of the perl5-porters@perl.org,
perl-unicode@perl.org, linux-utf8@nl.linux.org, and unicore@unicode.org
mailing lists for their valuable feedback.</p>
<a name="AUTHOR%2c-COPYRIGHT%2c-AND-LICENSE"></a><h1>AUTHOR, COPYRIGHT, AND LICENSE</h1>
<p>Copyright 2001-2011 Jarkko Hietaniemi <jhi@iki.fi>.
Now maintained by Perl 5 Porters.</p>
<p>This document may be distributed under the same terms as Perl itself.</p>
<div id="page_index" class="hud_container">
<div id="page_index_header" class="hud_header">
<div id="page_index_close" class="hud_close"><a href="#" onClick="pageIndex.hide();return false;"></a></div>
<div id="page_index_title" class="hud_title"><span class="hud_span_top">Page index</span></div>
<div id="page_index_topright" class="hud_topright"></div>
</div>
<div id="page_index_content" class="hud_content">
<ul><li><a href="#NAME">NAME</a><li><a href="#DESCRIPTION">DESCRIPTION</a><ul><li><a href="#Unicode">Unicode</a><li><a href="#Perl's-Unicode-Support">Perl's Unicode Support</a><li><a href="#Perl's-Unicode-Model">Perl's Unicode Model</a><li><a href="#Unicode-and-EBCDIC">Unicode and EBCDIC</a><li><a href="#Creating-Unicode">Creating Unicode</a><li><a href="#Handling-Unicode">Handling Unicode</a><li><a href="#Legacy-Encodings">Legacy Encodings</a><li><a href="#Unicode-I%2fO">Unicode I/O</a><li><a href="#Displaying-Unicode-As-Text">Displaying Unicode As Text</a><li><a href="#Special-Cases">Special Cases</a><li><a href="#Advanced-Topics">Advanced Topics</a><li><a href="#Miscellaneous">Miscellaneous</a><li><a href="#Questions-With-Answers">Questions With Answers</a><li><a href="#Hexadecimal-Notation">Hexadecimal Notation</a><li><a href="#Further-Resources">Further Resources</a></ul><li><a href="#UNICODE-IN-OLDER-PERLS">UNICODE IN OLDER PERLS</a><li><a href="#SEE-ALSO">SEE ALSO</a><li><a href="#ACKNOWLEDGMENTS">ACKNOWLEDGMENTS</a><li><a href="#AUTHOR%2c-COPYRIGHT%2c-AND-LICENSE">AUTHOR, COPYRIGHT, AND LICENSE</a></ul>
</div>
<div id="page_index_footer" class="hud_footer">
<div id="page_index_bottomleft" class="hud_bottomleft"></div>
<div id="page_index_bottom" class="hud_bottom"><span class="hud_span_bottom"></span></div>
<div id="page_index_resize" class="hud_resize"></div>
</div>
</div>
</div>
<div id="content_footer">
</div>
</div>
<div class="clear"></div>
</div>
<div id="footer">
<div id="footer_content">
<div id="footer_strapline">
perldoc.perl.org - Official documentation for the Perl programming language
</div>
<div id="footer_links">
<div id="address">
<p class="name">Contact details</p>
<p class="address">
Site maintained by <a href="mailto:jj@jonallen.info">Jon Allen (JJ)</a><br>
</p>
<p class="contact">
Documentation maintained by the <a href="http://lists.cpan.org/showlist.cgi?name=perl5-porters">Perl 5 Porters</a>
</p>
</div>
<ul class="f1">
<li>Manual
<ul class="f2">
<li><a href="index-overview.html">Overview</a>
<li><a href="index-tutorials.html">Tutorials</a>
<li><a href="index-faq.html">FAQs</a>
<li><a href="index-history.html">Changes</a>
</ul>
<li>Reference
<ul class="f2">
<li><a href="index-language.html">Language</a>
<li><a href="index-functions.html">Functions</a>
<li><a href="perlop.html">Operators</a>
<li><a href="perlvar.html">Variables</a>
</ul>
<li>Modules
<ul class="f2">
<li><a href="index-modules-A.html">Modules</a>
<li><a href="index-pragmas.html">Pragmas</a>
<li><a href="index-utilities.html">Utilities</a>
</ul>
<li>Misc
<ul class="f2">
<li><a href="index-licence.html">License</a>
<li><a href="index-internals.html">Internals</a>
<li><a href="index-platforms.html">Platforms</a>
</ul> </ul>
<div class="clear"></div>
</div>
</div>
<div id="footer_end">
</div>
</div>
</div>
<script language="JavaScript" type="text/javascript" src="static/exploreperl.js"></script>
<script language="JavaScript" src="static/combined-20100403.js" type="text/javascript"></script>
<script language="JavaScript" type="text/javascript">
perldoc.setPath(0);
perldoc.pageName = 'perluniintro';
perldoc.pageAddress = 'perluniintro.html';
perldoc.contentPage = 1;
explorePerl.render();
explorePerl.addEvents('explore_anchor');
</script>
</body>
</html>
|