Ruby 3.3.2p78 (2024-05-30 revision e5a195edf62fe1bf7146a191da13fa1c4fecbd71)
encoding.c
1/**********************************************************************
2
3 encoding.c -
4
5 $Author$
6 created at: Thu May 24 17:23:27 JST 2007
7
8 Copyright (C) 2007 Yukihiro Matsumoto
9
10**********************************************************************/
11
12#include "ruby/internal/config.h"
13
14#include <ctype.h>
15
16#include "encindex.h"
17#include "internal.h"
18#include "internal/enc.h"
19#include "internal/encoding.h"
20#include "internal/error.h"
21#include "internal/inits.h"
22#include "internal/load.h"
23#include "internal/object.h"
24#include "internal/string.h"
25#include "internal/vm.h"
26#include "regenc.h"
27#include "ruby/encoding.h"
28#include "ruby/util.h"
29#include "ruby_assert.h"
30#include "vm_sync.h"
31
32#ifndef ENC_DEBUG
33#define ENC_DEBUG 0
34#endif
35#define ENC_ASSERT(expr) RUBY_ASSERT_WHEN(ENC_DEBUG, expr)
36#define MUST_STRING(str) (ENC_ASSERT(RB_TYPE_P(str, T_STRING)), str)
37
38#undef rb_ascii8bit_encindex
39#undef rb_utf8_encindex
40#undef rb_usascii_encindex
41
43
44#if defined __GNUC__ && __GNUC__ >= 4
45#pragma GCC visibility push(default)
46int rb_enc_register(const char *name, rb_encoding *encoding);
47void rb_enc_set_base(const char *name, const char *orig);
48int rb_enc_set_dummy(int index);
49void rb_encdb_declare(const char *name);
50int rb_encdb_replicate(const char *name, const char *orig);
51int rb_encdb_dummy(const char *name);
52int rb_encdb_alias(const char *alias, const char *orig);
53#pragma GCC visibility pop
54#endif
55
56static ID id_encoding;
58
59#define ENCODING_LIST_CAPA 256
60static VALUE rb_encoding_list;
61
63 const char *name;
64 rb_encoding *enc;
65 rb_encoding *base;
66};
67
68static struct enc_table {
69 struct rb_encoding_entry list[ENCODING_LIST_CAPA];
70 int count;
71 st_table *names;
72} global_enc_table;
73
74static int
75enc_names_free_i(st_data_t name, st_data_t idx, st_data_t args)
76{
77 ruby_xfree((void *)name);
78 return ST_DELETE;
79}
80
81void
82rb_free_global_enc_table(void)
83{
84 for (size_t i = 0; i < ENCODING_LIST_CAPA; i++) {
85 xfree((void *)global_enc_table.list[i].enc);
86 }
87
88 st_foreach(global_enc_table.names, enc_names_free_i, (st_data_t)0);
89 st_free_table(global_enc_table.names);
90}
91
92static rb_encoding *global_enc_ascii,
93 *global_enc_utf_8,
94 *global_enc_us_ascii;
95
96#define GLOBAL_ENC_TABLE_ENTER(enc_table) struct enc_table *enc_table = &global_enc_table; RB_VM_LOCK_ENTER()
97#define GLOBAL_ENC_TABLE_LEAVE() RB_VM_LOCK_LEAVE()
98#define GLOBAL_ENC_TABLE_EVAL(enc_table, expr) do { \
99 GLOBAL_ENC_TABLE_ENTER(enc_table); \
100 { \
101 expr; \
102 } \
103 GLOBAL_ENC_TABLE_LEAVE(); \
104} while (0)
105
106
107#define ENC_DUMMY_FLAG (1<<24)
108#define ENC_INDEX_MASK (~(~0U<<24))
109
110#define ENC_TO_ENCINDEX(enc) (int)((enc)->ruby_encoding_index & ENC_INDEX_MASK)
111#define ENC_DUMMY_P(enc) ((enc)->ruby_encoding_index & ENC_DUMMY_FLAG)
112#define ENC_SET_DUMMY(enc) ((enc)->ruby_encoding_index |= ENC_DUMMY_FLAG)
113
114#define ENCODING_COUNT ENCINDEX_BUILTIN_MAX
115#define UNSPECIFIED_ENCODING INT_MAX
116
117#define ENCODING_NAMELEN_MAX 63
118#define valid_encoding_name_p(name) ((name) && strlen(name) <= ENCODING_NAMELEN_MAX)
119
120static const rb_data_type_t encoding_data_type = {
121 "encoding",
122 {0, 0, 0,},
123 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
124};
125
126#define is_data_encoding(obj) (RTYPEDDATA_P(obj) && RTYPEDDATA_TYPE(obj) == &encoding_data_type)
127#define is_obj_encoding(obj) (RB_TYPE_P((obj), T_DATA) && is_data_encoding(obj))
128
129int
130rb_data_is_encoding(VALUE obj)
131{
132 return is_data_encoding(obj);
133}
134
135static VALUE
136enc_new(rb_encoding *encoding)
137{
138 VALUE enc = TypedData_Wrap_Struct(rb_cEncoding, &encoding_data_type, (void *)encoding);
139 rb_obj_freeze(enc);
141 return enc;
142}
143
144static void
145enc_list_update(int index, rb_raw_encoding *encoding)
146{
147 RUBY_ASSERT(index < ENCODING_LIST_CAPA);
148
149 VALUE list = rb_encoding_list;
150 if (list && NIL_P(rb_ary_entry(list, index))) {
151 /* initialize encoding data */
152 rb_ary_store(list, index, enc_new(encoding));
153 }
154}
155
156static VALUE
157enc_list_lookup(int idx)
158{
159 VALUE list, enc = Qnil;
160
161 if (idx < ENCODING_LIST_CAPA) {
162 list = rb_encoding_list;
163 RUBY_ASSERT(list);
164 enc = rb_ary_entry(list, idx);
165 }
166
167 if (NIL_P(enc)) {
168 rb_bug("rb_enc_from_encoding_index(%d): not created yet", idx);
169 }
170 else {
171 return enc;
172 }
173}
174
175static VALUE
176rb_enc_from_encoding_index(int idx)
177{
178 return enc_list_lookup(idx);
179}
180
181VALUE
182rb_enc_from_encoding(rb_encoding *encoding)
183{
184 int idx;
185 if (!encoding) return Qnil;
186 idx = ENC_TO_ENCINDEX(encoding);
187 return rb_enc_from_encoding_index(idx);
188}
189
190int
191rb_enc_to_index(rb_encoding *enc)
192{
193 return enc ? ENC_TO_ENCINDEX(enc) : 0;
194}
195
196int
197rb_enc_dummy_p(rb_encoding *enc)
198{
199 return ENC_DUMMY_P(enc) != 0;
200}
201
202static int
203check_encoding(rb_encoding *enc)
204{
205 int index = rb_enc_to_index(enc);
206 if (rb_enc_from_index(index) != enc)
207 return -1;
208 if (rb_enc_autoload_p(enc)) {
209 index = rb_enc_autoload(enc);
210 }
211 return index;
212}
213
214static int
215enc_check_encoding(VALUE obj)
216{
217 if (!is_obj_encoding(obj)) {
218 return -1;
219 }
220 return check_encoding(RDATA(obj)->data);
221}
222
223NORETURN(static void not_encoding(VALUE enc));
224static void
225not_encoding(VALUE enc)
226{
227 rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected Encoding)",
228 rb_obj_class(enc));
229}
230
231static rb_encoding *
232must_encoding(VALUE enc)
233{
234 int index = enc_check_encoding(enc);
235 if (index < 0) {
236 not_encoding(enc);
237 }
238 return DATA_PTR(enc);
239}
240
241static rb_encoding *
242must_encindex(int index)
243{
244 rb_encoding *enc = rb_enc_from_index(index);
245 if (!enc) {
246 rb_raise(rb_eEncodingError, "encoding index out of bound: %d",
247 index);
248 }
249 if (ENC_TO_ENCINDEX(enc) != (int)(index & ENC_INDEX_MASK)) {
250 rb_raise(rb_eEncodingError, "wrong encoding index %d for %s (expected %d)",
251 index, rb_enc_name(enc), ENC_TO_ENCINDEX(enc));
252 }
253 if (rb_enc_autoload_p(enc) && rb_enc_autoload(enc) == -1) {
254 rb_loaderror("failed to load encoding (%s)",
255 rb_enc_name(enc));
256 }
257 return enc;
258}
259
260int
261rb_to_encoding_index(VALUE enc)
262{
263 int idx;
264 const char *name;
265
266 idx = enc_check_encoding(enc);
267 if (idx >= 0) {
268 return idx;
269 }
270 else if (NIL_P(enc = rb_check_string_type(enc))) {
271 return -1;
272 }
273 if (!rb_enc_asciicompat(rb_enc_get(enc))) {
274 return -1;
275 }
276 if (!(name = rb_str_to_cstr(enc))) {
277 return -1;
278 }
279 return rb_enc_find_index(name);
280}
281
282static const char *
283name_for_encoding(volatile VALUE *enc)
284{
285 VALUE name = StringValue(*enc);
286 const char *n;
287
288 if (!rb_enc_asciicompat(rb_enc_get(name))) {
289 rb_raise(rb_eArgError, "invalid encoding name (non ASCII)");
290 }
291 if (!(n = rb_str_to_cstr(name))) {
292 rb_raise(rb_eArgError, "invalid encoding name (NUL byte)");
293 }
294 return n;
295}
296
297/* Returns encoding index or UNSPECIFIED_ENCODING */
298static int
299str_find_encindex(VALUE enc)
300{
301 int idx = rb_enc_find_index(name_for_encoding(&enc));
302 RB_GC_GUARD(enc);
303 return idx;
304}
305
306static int
307str_to_encindex(VALUE enc)
308{
309 int idx = str_find_encindex(enc);
310 if (idx < 0) {
311 rb_raise(rb_eArgError, "unknown encoding name - %"PRIsVALUE, enc);
312 }
313 return idx;
314}
315
316static rb_encoding *
317str_to_encoding(VALUE enc)
318{
319 return rb_enc_from_index(str_to_encindex(enc));
320}
321
323rb_to_encoding(VALUE enc)
324{
325 if (enc_check_encoding(enc) >= 0) return RDATA(enc)->data;
326 return str_to_encoding(enc);
327}
328
330rb_find_encoding(VALUE enc)
331{
332 int idx;
333 if (enc_check_encoding(enc) >= 0) return RDATA(enc)->data;
334 idx = str_find_encindex(enc);
335 if (idx < 0) return NULL;
336 return rb_enc_from_index(idx);
337}
338
339static int
340enc_table_expand(struct enc_table *enc_table, int newsize)
341{
342 if (newsize > ENCODING_LIST_CAPA) {
343 rb_raise(rb_eEncodingError, "too many encoding (> %d)", ENCODING_LIST_CAPA);
344 }
345 return newsize;
346}
347
348static int
349enc_register_at(struct enc_table *enc_table, int index, const char *name, rb_encoding *base_encoding)
350{
351 struct rb_encoding_entry *ent = &enc_table->list[index];
352 rb_raw_encoding *encoding;
353
354 if (!valid_encoding_name_p(name)) return -1;
355 if (!ent->name) {
356 ent->name = name = strdup(name);
357 }
358 else if (STRCASECMP(name, ent->name)) {
359 return -1;
360 }
361 encoding = (rb_raw_encoding *)ent->enc;
362 if (!encoding) {
363 encoding = xmalloc(sizeof(rb_encoding));
364 }
365 if (base_encoding) {
366 *encoding = *base_encoding;
367 }
368 else {
369 memset(encoding, 0, sizeof(*ent->enc));
370 }
371 encoding->name = name;
372 encoding->ruby_encoding_index = index;
373 ent->enc = encoding;
374 st_insert(enc_table->names, (st_data_t)name, (st_data_t)index);
375
376 enc_list_update(index, encoding);
377 return index;
378}
379
380static int
381enc_register(struct enc_table *enc_table, const char *name, rb_encoding *encoding)
382{
383 int index = enc_table->count;
384
385 enc_table->count = enc_table_expand(enc_table, index + 1);
386 return enc_register_at(enc_table, index, name, encoding);
387}
388
389static void set_encoding_const(const char *, rb_encoding *);
390static int enc_registered(struct enc_table *enc_table, const char *name);
391
392static rb_encoding *
393enc_from_index(struct enc_table *enc_table, int index)
394{
395 if (UNLIKELY(index < 0 || enc_table->count <= (index &= ENC_INDEX_MASK))) {
396 return 0;
397 }
398 return enc_table->list[index].enc;
399}
400
402rb_enc_from_index(int index)
403{
404 return enc_from_index(&global_enc_table, index);
405}
406
407int
408rb_enc_register(const char *name, rb_encoding *encoding)
409{
410 int index;
411
412 GLOBAL_ENC_TABLE_ENTER(enc_table);
413 {
414 index = enc_registered(enc_table, name);
415
416 if (index >= 0) {
417 rb_encoding *oldenc = enc_from_index(enc_table, index);
418 if (STRCASECMP(name, rb_enc_name(oldenc))) {
419 index = enc_register(enc_table, name, encoding);
420 }
421 else if (rb_enc_autoload_p(oldenc) || !ENC_DUMMY_P(oldenc)) {
422 enc_register_at(enc_table, index, name, encoding);
423 }
424 else {
425 rb_raise(rb_eArgError, "encoding %s is already registered", name);
426 }
427 }
428 else {
429 index = enc_register(enc_table, name, encoding);
430 set_encoding_const(name, rb_enc_from_index(index));
431 }
432 }
433 GLOBAL_ENC_TABLE_LEAVE();
434 return index;
435}
436
437int
438enc_registered(struct enc_table *enc_table, const char *name)
439{
440 st_data_t idx = 0;
441
442 if (!name) return -1;
443 if (!enc_table->names) return -1;
444 if (st_lookup(enc_table->names, (st_data_t)name, &idx)) {
445 return (int)idx;
446 }
447 return -1;
448}
449
450void
451rb_encdb_declare(const char *name)
452{
453 GLOBAL_ENC_TABLE_ENTER(enc_table);
454 {
455 int idx = enc_registered(enc_table, name);
456 if (idx < 0) {
457 idx = enc_register(enc_table, name, 0);
458 }
459 set_encoding_const(name, rb_enc_from_index(idx));
460 }
461 GLOBAL_ENC_TABLE_LEAVE();
462}
463
464static void
465enc_check_addable(struct enc_table *enc_table, const char *name)
466{
467 if (enc_registered(enc_table, name) >= 0) {
468 rb_raise(rb_eArgError, "encoding %s is already registered", name);
469 }
470 else if (!valid_encoding_name_p(name)) {
471 rb_raise(rb_eArgError, "invalid encoding name: %s", name);
472 }
473}
474
475static rb_encoding*
476set_base_encoding(struct enc_table *enc_table, int index, rb_encoding *base)
477{
478 rb_encoding *enc = enc_table->list[index].enc;
479
480 ASSUME(enc);
481 enc_table->list[index].base = base;
482 if (ENC_DUMMY_P(base)) ENC_SET_DUMMY((rb_raw_encoding *)enc);
483 return enc;
484}
485
486/* for encdb.h
487 * Set base encoding for encodings which are not replicas
488 * but not in their own files.
489 */
490void
491rb_enc_set_base(const char *name, const char *orig)
492{
493 GLOBAL_ENC_TABLE_ENTER(enc_table);
494 {
495 int idx = enc_registered(enc_table, name);
496 int origidx = enc_registered(enc_table, orig);
497 set_base_encoding(enc_table, idx, rb_enc_from_index(origidx));
498 }
499 GLOBAL_ENC_TABLE_LEAVE();
500}
501
502/* for encdb.h
503 * Set encoding dummy.
504 */
505int
506rb_enc_set_dummy(int index)
507{
508 rb_encoding *enc = global_enc_table.list[index].enc;
509 ENC_SET_DUMMY((rb_raw_encoding *)enc);
510 return index;
511}
512
513static int
514enc_replicate(struct enc_table *enc_table, const char *name, rb_encoding *encoding)
515{
516 int idx;
517
518 enc_check_addable(enc_table, name);
519 idx = enc_register(enc_table, name, encoding);
520 if (idx < 0) rb_raise(rb_eArgError, "invalid encoding name: %s", name);
521 set_base_encoding(enc_table, idx, encoding);
522 set_encoding_const(name, rb_enc_from_index(idx));
523 return idx;
524}
525
526static int
527enc_replicate_with_index(struct enc_table *enc_table, const char *name, rb_encoding *origenc, int idx)
528{
529 if (idx < 0) {
530 idx = enc_register(enc_table, name, origenc);
531 }
532 else {
533 idx = enc_register_at(enc_table, idx, name, origenc);
534 }
535 if (idx >= 0) {
536 set_base_encoding(enc_table, idx, origenc);
537 set_encoding_const(name, rb_enc_from_index(idx));
538 }
539 else {
540 rb_raise(rb_eArgError, "failed to replicate encoding");
541 }
542 return idx;
543}
544
545int
546rb_encdb_replicate(const char *name, const char *orig)
547{
548 int r;
549
550 GLOBAL_ENC_TABLE_ENTER(enc_table);
551 {
552 int origidx = enc_registered(enc_table, orig);
553 int idx = enc_registered(enc_table, name);
554
555 if (origidx < 0) {
556 origidx = enc_register(enc_table, orig, 0);
557 }
558 r = enc_replicate_with_index(enc_table, name, rb_enc_from_index(origidx), idx);
559 }
560 GLOBAL_ENC_TABLE_LEAVE();
561
562 return r;
563}
564
565int
566rb_define_dummy_encoding(const char *name)
567{
568 int index;
569
570 GLOBAL_ENC_TABLE_ENTER(enc_table);
571 {
572 index = enc_replicate(enc_table, name, rb_ascii8bit_encoding());
573 rb_encoding *enc = enc_table->list[index].enc;
574 ENC_SET_DUMMY((rb_raw_encoding *)enc);
575 }
576 GLOBAL_ENC_TABLE_LEAVE();
577
578 return index;
579}
580
581int
582rb_encdb_dummy(const char *name)
583{
584 int index;
585
586 GLOBAL_ENC_TABLE_ENTER(enc_table);
587 {
588 index = enc_replicate_with_index(enc_table, name,
589 rb_ascii8bit_encoding(),
590 enc_registered(enc_table, name));
591 rb_encoding *enc = enc_table->list[index].enc;
592 ENC_SET_DUMMY((rb_raw_encoding *)enc);
593 }
594 GLOBAL_ENC_TABLE_LEAVE();
595
596 return index;
597}
598
599/*
600 * call-seq:
601 * enc.dummy? -> true or false
602 *
603 * Returns true for dummy encodings.
604 * A dummy encoding is an encoding for which character handling is not properly
605 * implemented.
606 * It is used for stateful encodings.
607 *
608 * Encoding::ISO_2022_JP.dummy? #=> true
609 * Encoding::UTF_8.dummy? #=> false
610 *
611 */
612static VALUE
613enc_dummy_p(VALUE enc)
614{
615 return RBOOL(ENC_DUMMY_P(must_encoding(enc)));
616}
617
618/*
619 * call-seq:
620 * enc.ascii_compatible? -> true or false
621 *
622 * Returns whether ASCII-compatible or not.
623 *
624 * Encoding::UTF_8.ascii_compatible? #=> true
625 * Encoding::UTF_16BE.ascii_compatible? #=> false
626 *
627 */
628static VALUE
629enc_ascii_compatible_p(VALUE enc)
630{
631 return RBOOL(rb_enc_asciicompat(must_encoding(enc)));
632}
633
634/*
635 * Returns non-zero when the encoding is Unicode series other than UTF-7 else 0.
636 */
637int
638rb_enc_unicode_p(rb_encoding *enc)
639{
640 return ONIGENC_IS_UNICODE(enc);
641}
642
643static st_data_t
644enc_dup_name(st_data_t name)
645{
646 return (st_data_t)strdup((const char *)name);
647}
648
649/*
650 * Returns copied alias name when the key is added for st_table,
651 * else returns NULL.
652 */
653static int
654enc_alias_internal(struct enc_table *enc_table, const char *alias, int idx)
655{
656 return st_insert2(enc_table->names, (st_data_t)alias, (st_data_t)idx,
657 enc_dup_name);
658}
659
660static int
661enc_alias(struct enc_table *enc_table, const char *alias, int idx)
662{
663 if (!valid_encoding_name_p(alias)) return -1;
664 if (!enc_alias_internal(enc_table, alias, idx))
665 set_encoding_const(alias, enc_from_index(enc_table, idx));
666 return idx;
667}
668
669int
670rb_enc_alias(const char *alias, const char *orig)
671{
672 int idx, r;
673
674 GLOBAL_ENC_TABLE_ENTER(enc_table);
675 {
676 enc_check_addable(enc_table, alias);
677 if ((idx = rb_enc_find_index(orig)) < 0) {
678 r = -1;
679 }
680 else {
681 r = enc_alias(enc_table, alias, idx);
682 }
683 }
684 GLOBAL_ENC_TABLE_LEAVE();
685
686 return r;
687}
688
689int
690rb_encdb_alias(const char *alias, const char *orig)
691{
692 int r;
693
694 GLOBAL_ENC_TABLE_ENTER(enc_table);
695 {
696 int idx = enc_registered(enc_table, orig);
697
698 if (idx < 0) {
699 idx = enc_register(enc_table, orig, 0);
700 }
701 r = enc_alias(enc_table, alias, idx);
702 }
703 GLOBAL_ENC_TABLE_LEAVE();
704
705 return r;
706}
707
708static void
709rb_enc_init(struct enc_table *enc_table)
710{
711 enc_table_expand(enc_table, ENCODING_COUNT + 1);
712 if (!enc_table->names) {
713 enc_table->names = st_init_strcasetable_with_size(ENCODING_LIST_CAPA);
714 }
715#define OnigEncodingASCII_8BIT OnigEncodingASCII
716#define ENC_REGISTER(enc) enc_register_at(enc_table, ENCINDEX_##enc, rb_enc_name(&OnigEncoding##enc), &OnigEncoding##enc)
717 ENC_REGISTER(ASCII_8BIT);
718 ENC_REGISTER(UTF_8);
719 ENC_REGISTER(US_ASCII);
720 global_enc_ascii = enc_table->list[ENCINDEX_ASCII_8BIT].enc;
721 global_enc_utf_8 = enc_table->list[ENCINDEX_UTF_8].enc;
722 global_enc_us_ascii = enc_table->list[ENCINDEX_US_ASCII].enc;
723#undef ENC_REGISTER
724#undef OnigEncodingASCII_8BIT
725#define ENCDB_REGISTER(name, enc) enc_register_at(enc_table, ENCINDEX_##enc, name, NULL)
726 ENCDB_REGISTER("UTF-16BE", UTF_16BE);
727 ENCDB_REGISTER("UTF-16LE", UTF_16LE);
728 ENCDB_REGISTER("UTF-32BE", UTF_32BE);
729 ENCDB_REGISTER("UTF-32LE", UTF_32LE);
730 ENCDB_REGISTER("UTF-16", UTF_16);
731 ENCDB_REGISTER("UTF-32", UTF_32);
732 ENCDB_REGISTER("UTF8-MAC", UTF8_MAC);
733
734 ENCDB_REGISTER("EUC-JP", EUC_JP);
735 ENCDB_REGISTER("Windows-31J", Windows_31J);
736#undef ENCDB_REGISTER
737 enc_table->count = ENCINDEX_BUILTIN_MAX;
738}
739
741rb_enc_get_from_index(int index)
742{
743 return must_encindex(index);
744}
745
746int rb_require_internal_silent(VALUE fname);
747
748static int
749load_encoding(const char *name)
750{
751 VALUE enclib = rb_sprintf("enc/%s.so", name);
752 VALUE debug = ruby_debug;
753 VALUE errinfo;
754 char *s = RSTRING_PTR(enclib) + 4, *e = RSTRING_END(enclib) - 3;
755 int loaded;
756 int idx;
757
758 while (s < e) {
759 if (!ISALNUM(*s)) *s = '_';
760 else if (ISUPPER(*s)) *s = (char)TOLOWER(*s);
761 ++s;
762 }
763 enclib = rb_fstring(enclib);
765 errinfo = rb_errinfo();
766 loaded = rb_require_internal_silent(enclib);
767 ruby_debug = debug;
768 rb_set_errinfo(errinfo);
769
770 GLOBAL_ENC_TABLE_ENTER(enc_table);
771 {
772 if (loaded < 0 || 1 < loaded) {
773 idx = -1;
774 }
775 else if ((idx = enc_registered(enc_table, name)) < 0) {
776 idx = -1;
777 }
778 else if (rb_enc_autoload_p(enc_table->list[idx].enc)) {
779 idx = -1;
780 }
781 }
782 GLOBAL_ENC_TABLE_LEAVE();
783
784 return idx;
785}
786
787static int
788enc_autoload_body(struct enc_table *enc_table, rb_encoding *enc)
789{
790 rb_encoding *base = enc_table->list[ENC_TO_ENCINDEX(enc)].base;
791
792 if (base) {
793 int i = 0;
794 do {
795 if (i >= enc_table->count) return -1;
796 } while (enc_table->list[i].enc != base && (++i, 1));
797 if (rb_enc_autoload_p(base)) {
798 if (rb_enc_autoload(base) < 0) return -1;
799 }
800 i = enc->ruby_encoding_index;
801 enc_register_at(enc_table, i & ENC_INDEX_MASK, rb_enc_name(enc), base);
802 ((rb_raw_encoding *)enc)->ruby_encoding_index = i;
803 i &= ENC_INDEX_MASK;
804 return i;
805 }
806 else {
807 return -2;
808 }
809}
810
811int
812rb_enc_autoload(rb_encoding *enc)
813{
814 int i;
815 GLOBAL_ENC_TABLE_EVAL(enc_table, i = enc_autoload_body(enc_table, enc));
816 if (i == -2) {
817 i = load_encoding(rb_enc_name(enc));
818 }
819 return i;
820}
821
822/* Return encoding index or UNSPECIFIED_ENCODING from encoding name */
823int
824rb_enc_find_index(const char *name)
825{
826 int i = enc_registered(&global_enc_table, name);
827 rb_encoding *enc;
828
829 if (i < 0) {
830 i = load_encoding(name);
831 }
832 else if (!(enc = rb_enc_from_index(i))) {
833 if (i != UNSPECIFIED_ENCODING) {
834 rb_raise(rb_eArgError, "encoding %s is not registered", name);
835 }
836 }
837 else if (rb_enc_autoload_p(enc)) {
838 if (rb_enc_autoload(enc) < 0) {
839 rb_warn("failed to load encoding (%s); use ASCII-8BIT instead",
840 name);
841 return 0;
842 }
843 }
844 return i;
845}
846
847int
848rb_enc_find_index2(const char *name, long len)
849{
850 char buf[ENCODING_NAMELEN_MAX+1];
851
852 if (len > ENCODING_NAMELEN_MAX) return -1;
853 memcpy(buf, name, len);
854 buf[len] = '\0';
855 return rb_enc_find_index(buf);
856}
857
859rb_enc_find(const char *name)
860{
861 int idx = rb_enc_find_index(name);
862 if (idx < 0) idx = 0;
863 return rb_enc_from_index(idx);
864}
865
866static inline int
867enc_capable(VALUE obj)
868{
869 if (SPECIAL_CONST_P(obj)) return SYMBOL_P(obj);
870 switch (BUILTIN_TYPE(obj)) {
871 case T_STRING:
872 case T_REGEXP:
873 case T_FILE:
874 case T_SYMBOL:
875 return TRUE;
876 case T_DATA:
877 if (is_data_encoding(obj)) return TRUE;
878 default:
879 return FALSE;
880 }
881}
882
883int
884rb_enc_capable(VALUE obj)
885{
886 return enc_capable(obj);
887}
888
889ID
890rb_id_encoding(void)
891{
892 CONST_ID(id_encoding, "encoding");
893 return id_encoding;
894}
895
896static int
897enc_get_index_str(VALUE str)
898{
899 int i = ENCODING_GET_INLINED(str);
900 if (i == ENCODING_INLINE_MAX) {
901 VALUE iv;
902
903#if 0
904 iv = rb_ivar_get(str, rb_id_encoding());
905 i = NUM2INT(iv);
906#else
907 /*
908 * Tentatively, assume ASCII-8BIT, if encoding index instance
909 * variable is not found. This can happen when freeing after
910 * all instance variables are removed in `obj_free`.
911 */
912 iv = rb_attr_get(str, rb_id_encoding());
913 i = NIL_P(iv) ? ENCINDEX_ASCII_8BIT : NUM2INT(iv);
914#endif
915 }
916 return i;
917}
918
919int
920rb_enc_get_index(VALUE obj)
921{
922 int i = -1;
923 VALUE tmp;
924
925 if (SPECIAL_CONST_P(obj)) {
926 if (!SYMBOL_P(obj)) return -1;
927 obj = rb_sym2str(obj);
928 }
929 switch (BUILTIN_TYPE(obj)) {
930 case T_STRING:
931 case T_SYMBOL:
932 case T_REGEXP:
933 i = enc_get_index_str(obj);
934 break;
935 case T_FILE:
936 tmp = rb_funcallv(obj, rb_intern("internal_encoding"), 0, 0);
937 if (NIL_P(tmp)) {
938 tmp = rb_funcallv(obj, rb_intern("external_encoding"), 0, 0);
939 }
940 if (is_obj_encoding(tmp)) {
941 i = enc_check_encoding(tmp);
942 }
943 break;
944 case T_DATA:
945 if (is_data_encoding(obj)) {
946 i = enc_check_encoding(obj);
947 }
948 break;
949 default:
950 break;
951 }
952 return i;
953}
954
955static void
956enc_set_index(VALUE obj, int idx)
957{
958 if (!enc_capable(obj)) {
959 rb_raise(rb_eArgError, "cannot set encoding on non-encoding capable object");
960 }
961
962 if (idx < ENCODING_INLINE_MAX) {
963 ENCODING_SET_INLINED(obj, idx);
964 return;
965 }
967 rb_ivar_set(obj, rb_id_encoding(), INT2NUM(idx));
968}
969
970void
971rb_enc_set_index(VALUE obj, int idx)
972{
973 rb_check_frozen(obj);
974 must_encindex(idx);
975 enc_set_index(obj, idx);
976}
977
978VALUE
979rb_enc_associate_index(VALUE obj, int idx)
980{
981 rb_encoding *enc;
982 int oldidx, oldtermlen, termlen;
983
984/* enc_check_capable(obj);*/
985 rb_check_frozen(obj);
986 oldidx = rb_enc_get_index(obj);
987 if (oldidx == idx)
988 return obj;
989 if (SPECIAL_CONST_P(obj)) {
990 rb_raise(rb_eArgError, "cannot set encoding");
991 }
992 enc = must_encindex(idx);
993 if (!ENC_CODERANGE_ASCIIONLY(obj) ||
994 !rb_enc_asciicompat(enc)) {
996 }
997 termlen = rb_enc_mbminlen(enc);
998 oldtermlen = rb_enc_mbminlen(rb_enc_from_index(oldidx));
999 if (oldtermlen != termlen && RB_TYPE_P(obj, T_STRING)) {
1000 rb_str_change_terminator_length(obj, oldtermlen, termlen);
1001 }
1002 enc_set_index(obj, idx);
1003 return obj;
1004}
1005
1006VALUE
1007rb_enc_associate(VALUE obj, rb_encoding *enc)
1008{
1009 return rb_enc_associate_index(obj, rb_enc_to_index(enc));
1010}
1011
1013rb_enc_get(VALUE obj)
1014{
1015 return rb_enc_from_index(rb_enc_get_index(obj));
1016}
1017
1018static rb_encoding*
1019rb_encoding_check(rb_encoding* enc, VALUE str1, VALUE str2)
1020{
1021 if (!enc)
1022 rb_raise(rb_eEncCompatError, "incompatible character encodings: %s and %s",
1023 rb_enc_name(rb_enc_get(str1)),
1024 rb_enc_name(rb_enc_get(str2)));
1025 return enc;
1026}
1027
1028static rb_encoding* enc_compatible_str(VALUE str1, VALUE str2);
1029
1031rb_enc_check_str(VALUE str1, VALUE str2)
1032{
1033 rb_encoding *enc = enc_compatible_str(MUST_STRING(str1), MUST_STRING(str2));
1034 return rb_encoding_check(enc, str1, str2);
1035}
1036
1038rb_enc_check(VALUE str1, VALUE str2)
1039{
1040 rb_encoding *enc = rb_enc_compatible(str1, str2);
1041 return rb_encoding_check(enc, str1, str2);
1042}
1043
1044static rb_encoding*
1045enc_compatible_latter(VALUE str1, VALUE str2, int idx1, int idx2)
1046{
1047 int isstr1, isstr2;
1048 rb_encoding *enc1 = rb_enc_from_index(idx1);
1049 rb_encoding *enc2 = rb_enc_from_index(idx2);
1050
1051 isstr2 = RB_TYPE_P(str2, T_STRING);
1052 if (isstr2 && RSTRING_LEN(str2) == 0)
1053 return enc1;
1054 isstr1 = RB_TYPE_P(str1, T_STRING);
1055 if (isstr1 && isstr2 && RSTRING_LEN(str1) == 0)
1056 return (rb_enc_asciicompat(enc1) && rb_enc_str_asciionly_p(str2)) ? enc1 : enc2;
1057 if (!rb_enc_asciicompat(enc1) || !rb_enc_asciicompat(enc2)) {
1058 return 0;
1059 }
1060
1061 /* objects whose encoding is the same of contents */
1062 if (!isstr2 && idx2 == ENCINDEX_US_ASCII)
1063 return enc1;
1064 if (!isstr1 && idx1 == ENCINDEX_US_ASCII)
1065 return enc2;
1066
1067 if (!isstr1) {
1068 VALUE tmp = str1;
1069 int idx0 = idx1;
1070 str1 = str2;
1071 str2 = tmp;
1072 idx1 = idx2;
1073 idx2 = idx0;
1074 idx0 = isstr1;
1075 isstr1 = isstr2;
1076 isstr2 = idx0;
1077 }
1078 if (isstr1) {
1079 int cr1, cr2;
1080
1081 cr1 = rb_enc_str_coderange(str1);
1082 if (isstr2) {
1083 cr2 = rb_enc_str_coderange(str2);
1084 if (cr1 != cr2) {
1085 /* may need to handle ENC_CODERANGE_BROKEN */
1086 if (cr1 == ENC_CODERANGE_7BIT) return enc2;
1087 if (cr2 == ENC_CODERANGE_7BIT) return enc1;
1088 }
1089 if (cr2 == ENC_CODERANGE_7BIT) {
1090 return enc1;
1091 }
1092 }
1093 if (cr1 == ENC_CODERANGE_7BIT)
1094 return enc2;
1095 }
1096 return 0;
1097}
1098
1099static rb_encoding*
1100enc_compatible_str(VALUE str1, VALUE str2)
1101{
1102 int idx1 = enc_get_index_str(str1);
1103 int idx2 = enc_get_index_str(str2);
1104
1105 if (idx1 < 0 || idx2 < 0)
1106 return 0;
1107
1108 if (idx1 == idx2) {
1109 return rb_enc_from_index(idx1);
1110 }
1111 else {
1112 return enc_compatible_latter(str1, str2, idx1, idx2);
1113 }
1114}
1115
1117rb_enc_compatible(VALUE str1, VALUE str2)
1118{
1119 int idx1 = rb_enc_get_index(str1);
1120 int idx2 = rb_enc_get_index(str2);
1121
1122 if (idx1 < 0 || idx2 < 0)
1123 return 0;
1124
1125 if (idx1 == idx2) {
1126 return rb_enc_from_index(idx1);
1127 }
1128
1129 return enc_compatible_latter(str1, str2, idx1, idx2);
1130}
1131
1132void
1133rb_enc_copy(VALUE obj1, VALUE obj2)
1134{
1135 rb_enc_associate_index(obj1, rb_enc_get_index(obj2));
1136}
1137
1138
1139/*
1140 * call-seq:
1141 * obj.encoding -> encoding
1142 *
1143 * Returns the Encoding object that represents the encoding of obj.
1144 */
1145
1146VALUE
1147rb_obj_encoding(VALUE obj)
1148{
1149 int idx = rb_enc_get_index(obj);
1150 if (idx < 0) {
1151 rb_raise(rb_eTypeError, "unknown encoding");
1152 }
1153 return rb_enc_from_encoding_index(idx & ENC_INDEX_MASK);
1154}
1155
1156int
1157rb_enc_fast_mbclen(const char *p, const char *e, rb_encoding *enc)
1158{
1159 return ONIGENC_MBC_ENC_LEN(enc, (UChar*)p, (UChar*)e);
1160}
1161
1162int
1163rb_enc_mbclen(const char *p, const char *e, rb_encoding *enc)
1164{
1165 int n = ONIGENC_PRECISE_MBC_ENC_LEN(enc, (UChar*)p, (UChar*)e);
1166 if (MBCLEN_CHARFOUND_P(n) && MBCLEN_CHARFOUND_LEN(n) <= e-p)
1167 return MBCLEN_CHARFOUND_LEN(n);
1168 else {
1169 int min = rb_enc_mbminlen(enc);
1170 return min <= e-p ? min : (int)(e-p);
1171 }
1172}
1173
1174int
1175rb_enc_precise_mbclen(const char *p, const char *e, rb_encoding *enc)
1176{
1177 int n;
1178 if (e <= p)
1179 return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(1);
1180 n = ONIGENC_PRECISE_MBC_ENC_LEN(enc, (UChar*)p, (UChar*)e);
1181 if (e-p < n)
1182 return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(n-(int)(e-p));
1183 return n;
1184}
1185
1186int
1187rb_enc_ascget(const char *p, const char *e, int *len, rb_encoding *enc)
1188{
1189 unsigned int c;
1190 int l;
1191 if (e <= p)
1192 return -1;
1193 if (rb_enc_asciicompat(enc)) {
1194 c = (unsigned char)*p;
1195 if (!ISASCII(c))
1196 return -1;
1197 if (len) *len = 1;
1198 return c;
1199 }
1200 l = rb_enc_precise_mbclen(p, e, enc);
1201 if (!MBCLEN_CHARFOUND_P(l))
1202 return -1;
1203 c = rb_enc_mbc_to_codepoint(p, e, enc);
1204 if (!rb_enc_isascii(c, enc))
1205 return -1;
1206 if (len) *len = l;
1207 return c;
1208}
1209
1210unsigned int
1211rb_enc_codepoint_len(const char *p, const char *e, int *len_p, rb_encoding *enc)
1212{
1213 int r;
1214 if (e <= p)
1215 rb_raise(rb_eArgError, "empty string");
1216 r = rb_enc_precise_mbclen(p, e, enc);
1217 if (!MBCLEN_CHARFOUND_P(r)) {
1218 rb_raise(rb_eArgError, "invalid byte sequence in %s", rb_enc_name(enc));
1219 }
1220 if (len_p) *len_p = MBCLEN_CHARFOUND_LEN(r);
1221 return rb_enc_mbc_to_codepoint(p, e, enc);
1222}
1223
1224int
1225rb_enc_codelen(int c, rb_encoding *enc)
1226{
1227 int n = ONIGENC_CODE_TO_MBCLEN(enc,c);
1228 if (n == 0) {
1229 rb_raise(rb_eArgError, "invalid codepoint 0x%x in %s", c, rb_enc_name(enc));
1230 }
1231 return n;
1232}
1233
1234int
1235rb_enc_toupper(int c, rb_encoding *enc)
1236{
1237 return (ONIGENC_IS_ASCII_CODE(c)?ONIGENC_ASCII_CODE_TO_UPPER_CASE(c):(c));
1238}
1239
1240int
1241rb_enc_tolower(int c, rb_encoding *enc)
1242{
1243 return (ONIGENC_IS_ASCII_CODE(c)?ONIGENC_ASCII_CODE_TO_LOWER_CASE(c):(c));
1244}
1245
1246/*
1247 * call-seq:
1248 * enc.inspect -> string
1249 *
1250 * Returns a string which represents the encoding for programmers.
1251 *
1252 * Encoding::UTF_8.inspect #=> "#<Encoding:UTF-8>"
1253 * Encoding::ISO_2022_JP.inspect #=> "#<Encoding:ISO-2022-JP (dummy)>"
1254 */
1255static VALUE
1256enc_inspect(VALUE self)
1257{
1258 rb_encoding *enc;
1259
1260 if (!is_data_encoding(self)) {
1261 not_encoding(self);
1262 }
1263 if (!(enc = DATA_PTR(self)) || rb_enc_from_index(rb_enc_to_index(enc)) != enc) {
1264 rb_raise(rb_eTypeError, "broken Encoding");
1265 }
1266 return rb_enc_sprintf(rb_usascii_encoding(),
1267 "#<%"PRIsVALUE":%s%s%s>", rb_obj_class(self),
1268 rb_enc_name(enc),
1269 (ENC_DUMMY_P(enc) ? " (dummy)" : ""),
1270 rb_enc_autoload_p(enc) ? " (autoload)" : "");
1271}
1272
1273/*
1274 * call-seq:
1275 * enc.name -> string
1276 * enc.to_s -> string
1277 *
1278 * Returns the name of the encoding.
1279 *
1280 * Encoding::UTF_8.name #=> "UTF-8"
1281 */
1282static VALUE
1283enc_name(VALUE self)
1284{
1285 return rb_fstring_cstr(rb_enc_name((rb_encoding*)DATA_PTR(self)));
1286}
1287
1288static int
1289enc_names_i(st_data_t name, st_data_t idx, st_data_t args)
1290{
1291 VALUE *arg = (VALUE *)args;
1292
1293 if ((int)idx == (int)arg[0]) {
1294 VALUE str = rb_fstring_cstr((char *)name);
1295 rb_ary_push(arg[1], str);
1296 }
1297 return ST_CONTINUE;
1298}
1299
1300/*
1301 * call-seq:
1302 * enc.names -> array
1303 *
1304 * Returns the list of name and aliases of the encoding.
1305 *
1306 * Encoding::WINDOWS_31J.names #=> ["Windows-31J", "CP932", "csWindows31J", "SJIS", "PCK"]
1307 */
1308static VALUE
1309enc_names(VALUE self)
1310{
1311 VALUE args[2];
1312
1313 args[0] = (VALUE)rb_to_encoding_index(self);
1314 args[1] = rb_ary_new2(0);
1315 st_foreach(global_enc_table.names, enc_names_i, (st_data_t)args);
1316 return args[1];
1317}
1318
1319/*
1320 * call-seq:
1321 * Encoding.list -> [enc1, enc2, ...]
1322 *
1323 * Returns the list of loaded encodings.
1324 *
1325 * Encoding.list
1326 * #=> [#<Encoding:ASCII-8BIT>, #<Encoding:UTF-8>,
1327 * #<Encoding:ISO-2022-JP (dummy)>]
1328 *
1329 * Encoding.find("US-ASCII")
1330 * #=> #<Encoding:US-ASCII>
1331 *
1332 * Encoding.list
1333 * #=> [#<Encoding:ASCII-8BIT>, #<Encoding:UTF-8>,
1334 * #<Encoding:US-ASCII>, #<Encoding:ISO-2022-JP (dummy)>]
1335 *
1336 */
1337static VALUE
1338enc_list(VALUE klass)
1339{
1340 VALUE ary = rb_ary_new2(0);
1341 rb_ary_replace(ary, rb_encoding_list);
1342 return ary;
1343}
1344
1345/*
1346 * call-seq:
1347 * Encoding.find(string) -> enc
1348 *
1349 * Search the encoding with specified <i>name</i>.
1350 * <i>name</i> should be a string.
1351 *
1352 * Encoding.find("US-ASCII") #=> #<Encoding:US-ASCII>
1353 *
1354 * Names which this method accept are encoding names and aliases
1355 * including following special aliases
1356 *
1357 * "external":: default external encoding
1358 * "internal":: default internal encoding
1359 * "locale":: locale encoding
1360 * "filesystem":: filesystem encoding
1361 *
1362 * An ArgumentError is raised when no encoding with <i>name</i>.
1363 * Only <code>Encoding.find("internal")</code> however returns nil
1364 * when no encoding named "internal", in other words, when Ruby has no
1365 * default internal encoding.
1366 */
1367static VALUE
1368enc_find(VALUE klass, VALUE enc)
1369{
1370 int idx;
1371 if (is_obj_encoding(enc))
1372 return enc;
1373 idx = str_to_encindex(enc);
1374 if (idx == UNSPECIFIED_ENCODING) return Qnil;
1375 return rb_enc_from_encoding_index(idx);
1376}
1377
1378/*
1379 * call-seq:
1380 * Encoding.compatible?(obj1, obj2) -> enc or nil
1381 *
1382 * Checks the compatibility of two objects.
1383 *
1384 * If the objects are both strings they are compatible when they are
1385 * concatenatable. The encoding of the concatenated string will be returned
1386 * if they are compatible, nil if they are not.
1387 *
1388 * Encoding.compatible?("\xa1".force_encoding("iso-8859-1"), "b")
1389 * #=> #<Encoding:ISO-8859-1>
1390 *
1391 * Encoding.compatible?(
1392 * "\xa1".force_encoding("iso-8859-1"),
1393 * "\xa1\xa1".force_encoding("euc-jp"))
1394 * #=> nil
1395 *
1396 * If the objects are non-strings their encodings are compatible when they
1397 * have an encoding and:
1398 * * Either encoding is US-ASCII compatible
1399 * * One of the encodings is a 7-bit encoding
1400 *
1401 */
1402static VALUE
1403enc_compatible_p(VALUE klass, VALUE str1, VALUE str2)
1404{
1405 rb_encoding *enc;
1406
1407 if (!enc_capable(str1)) return Qnil;
1408 if (!enc_capable(str2)) return Qnil;
1409 enc = rb_enc_compatible(str1, str2);
1410 if (!enc) return Qnil;
1411 return rb_enc_from_encoding(enc);
1412}
1413
1414NORETURN(static VALUE enc_s_alloc(VALUE klass));
1415/* :nodoc: */
1416static VALUE
1417enc_s_alloc(VALUE klass)
1418{
1419 rb_undefined_alloc(klass);
1421}
1422
1423/* :nodoc: */
1424static VALUE
1425enc_dump(int argc, VALUE *argv, VALUE self)
1426{
1427 rb_check_arity(argc, 0, 1);
1428 return enc_name(self);
1429}
1430
1431/* :nodoc: */
1432static VALUE
1433enc_load(VALUE klass, VALUE str)
1434{
1435 return str;
1436}
1437
1438/* :nodoc: */
1439static VALUE
1440enc_m_loader(VALUE klass, VALUE str)
1441{
1442 return enc_find(klass, str);
1443}
1444
1446rb_ascii8bit_encoding(void)
1447{
1448 return global_enc_ascii;
1449}
1450
1451int
1452rb_ascii8bit_encindex(void)
1453{
1454 return ENCINDEX_ASCII_8BIT;
1455}
1456
1458rb_utf8_encoding(void)
1459{
1460 return global_enc_utf_8;
1461}
1462
1463int
1464rb_utf8_encindex(void)
1465{
1466 return ENCINDEX_UTF_8;
1467}
1468
1470rb_usascii_encoding(void)
1471{
1472 return global_enc_us_ascii;
1473}
1474
1475int
1476rb_usascii_encindex(void)
1477{
1478 return ENCINDEX_US_ASCII;
1479}
1480
1481int rb_locale_charmap_index(void);
1482
1483int
1484rb_locale_encindex(void)
1485{
1486 int idx = rb_locale_charmap_index();
1487
1488 if (idx < 0) idx = ENCINDEX_UTF_8;
1489
1490 if (enc_registered(&global_enc_table, "locale") < 0) {
1491# if defined _WIN32
1492 void Init_w32_codepage(void);
1493 Init_w32_codepage();
1494# endif
1495 GLOBAL_ENC_TABLE_ENTER(enc_table);
1496 {
1497 enc_alias_internal(enc_table, "locale", idx);
1498 }
1499 GLOBAL_ENC_TABLE_LEAVE();
1500 }
1501
1502 return idx;
1503}
1504
1506rb_locale_encoding(void)
1507{
1508 return rb_enc_from_index(rb_locale_encindex());
1509}
1510
1511int
1512rb_filesystem_encindex(void)
1513{
1514 int idx = enc_registered(&global_enc_table, "filesystem");
1515 if (idx < 0) idx = ENCINDEX_ASCII_8BIT;
1516 return idx;
1517}
1518
1520rb_filesystem_encoding(void)
1521{
1522 return rb_enc_from_index(rb_filesystem_encindex());
1523}
1524
1526 int index; /* -2 => not yet set, -1 => nil */
1527 rb_encoding *enc;
1528};
1529
1530static struct default_encoding default_external = {0};
1531
1532static int
1533enc_set_default_encoding(struct default_encoding *def, VALUE encoding, const char *name)
1534{
1535 int overridden = FALSE;
1536
1537 if (def->index != -2)
1538 /* Already set */
1539 overridden = TRUE;
1540
1541 GLOBAL_ENC_TABLE_ENTER(enc_table);
1542 {
1543 if (NIL_P(encoding)) {
1544 def->index = -1;
1545 def->enc = 0;
1546 char *name_dup = strdup(name);
1547
1548 st_data_t existing_name = (st_data_t)name_dup;
1549 if (st_delete(enc_table->names, &existing_name, NULL)) {
1550 xfree((void *)existing_name);
1551 }
1552
1553 st_insert(enc_table->names, (st_data_t)name_dup,
1554 (st_data_t)UNSPECIFIED_ENCODING);
1555 }
1556 else {
1557 def->index = rb_enc_to_index(rb_to_encoding(encoding));
1558 def->enc = 0;
1559 enc_alias_internal(enc_table, name, def->index);
1560 }
1561
1562 if (def == &default_external) {
1563 enc_alias_internal(enc_table, "filesystem", Init_enc_set_filesystem_encoding());
1564 }
1565 }
1566 GLOBAL_ENC_TABLE_LEAVE();
1567
1568 return overridden;
1569}
1570
1572rb_default_external_encoding(void)
1573{
1574 if (default_external.enc) return default_external.enc;
1575
1576 if (default_external.index >= 0) {
1577 default_external.enc = rb_enc_from_index(default_external.index);
1578 return default_external.enc;
1579 }
1580 else {
1581 return rb_locale_encoding();
1582 }
1583}
1584
1585VALUE
1586rb_enc_default_external(void)
1587{
1588 return rb_enc_from_encoding(rb_default_external_encoding());
1589}
1590
1591/*
1592 * call-seq:
1593 * Encoding.default_external -> enc
1594 *
1595 * Returns default external encoding.
1596 *
1597 * The default external encoding is used by default for strings created from
1598 * the following locations:
1599 *
1600 * * CSV
1601 * * File data read from disk
1602 * * SDBM
1603 * * StringIO
1604 * * Zlib::GzipReader
1605 * * Zlib::GzipWriter
1606 * * String#inspect
1607 * * Regexp#inspect
1608 *
1609 * While strings created from these locations will have this encoding, the
1610 * encoding may not be valid. Be sure to check String#valid_encoding?.
1611 *
1612 * File data written to disk will be transcoded to the default external
1613 * encoding when written, if default_internal is not nil.
1614 *
1615 * The default external encoding is initialized by the -E option.
1616 * If -E isn't set, it is initialized to UTF-8 on Windows and the locale on
1617 * other operating systems.
1618 */
1619static VALUE
1620get_default_external(VALUE klass)
1621{
1622 return rb_enc_default_external();
1623}
1624
1625void
1626rb_enc_set_default_external(VALUE encoding)
1627{
1628 if (NIL_P(encoding)) {
1629 rb_raise(rb_eArgError, "default external can not be nil");
1630 }
1631 enc_set_default_encoding(&default_external, encoding,
1632 "external");
1633}
1634
1635/*
1636 * call-seq:
1637 * Encoding.default_external = enc
1638 *
1639 * Sets default external encoding. You should not set
1640 * Encoding::default_external in ruby code as strings created before changing
1641 * the value may have a different encoding from strings created after the value
1642 * was changed., instead you should use <tt>ruby -E</tt> to invoke ruby with
1643 * the correct default_external.
1644 *
1645 * See Encoding::default_external for information on how the default external
1646 * encoding is used.
1647 */
1648static VALUE
1649set_default_external(VALUE klass, VALUE encoding)
1650{
1651 rb_warning("setting Encoding.default_external");
1652 rb_enc_set_default_external(encoding);
1653 return encoding;
1654}
1655
1656static struct default_encoding default_internal = {-2};
1657
1659rb_default_internal_encoding(void)
1660{
1661 if (!default_internal.enc && default_internal.index >= 0) {
1662 default_internal.enc = rb_enc_from_index(default_internal.index);
1663 }
1664 return default_internal.enc; /* can be NULL */
1665}
1666
1667VALUE
1668rb_enc_default_internal(void)
1669{
1670 /* Note: These functions cope with default_internal not being set */
1671 return rb_enc_from_encoding(rb_default_internal_encoding());
1672}
1673
1674/*
1675 * call-seq:
1676 * Encoding.default_internal -> enc
1677 *
1678 * Returns default internal encoding. Strings will be transcoded to the
1679 * default internal encoding in the following places if the default internal
1680 * encoding is not nil:
1681 *
1682 * * CSV
1683 * * Etc.sysconfdir and Etc.systmpdir
1684 * * File data read from disk
1685 * * File names from Dir
1686 * * Integer#chr
1687 * * String#inspect and Regexp#inspect
1688 * * Strings returned from Readline
1689 * * Strings returned from SDBM
1690 * * Time#zone
1691 * * Values from ENV
1692 * * Values in ARGV including $PROGRAM_NAME
1693 *
1694 * Additionally String#encode and String#encode! use the default internal
1695 * encoding if no encoding is given.
1696 *
1697 * The script encoding (__ENCODING__), not default_internal, is used as the
1698 * encoding of created strings.
1699 *
1700 * Encoding::default_internal is initialized with -E option or nil otherwise.
1701 */
1702static VALUE
1703get_default_internal(VALUE klass)
1704{
1705 return rb_enc_default_internal();
1706}
1707
1708void
1709rb_enc_set_default_internal(VALUE encoding)
1710{
1711 enc_set_default_encoding(&default_internal, encoding,
1712 "internal");
1713}
1714
1715/*
1716 * call-seq:
1717 * Encoding.default_internal = enc or nil
1718 *
1719 * Sets default internal encoding or removes default internal encoding when
1720 * passed nil. You should not set Encoding::default_internal in ruby code as
1721 * strings created before changing the value may have a different encoding
1722 * from strings created after the change. Instead you should use
1723 * <tt>ruby -E</tt> to invoke ruby with the correct default_internal.
1724 *
1725 * See Encoding::default_internal for information on how the default internal
1726 * encoding is used.
1727 */
1728static VALUE
1729set_default_internal(VALUE klass, VALUE encoding)
1730{
1731 rb_warning("setting Encoding.default_internal");
1732 rb_enc_set_default_internal(encoding);
1733 return encoding;
1734}
1735
1736static void
1737set_encoding_const(const char *name, rb_encoding *enc)
1738{
1739 VALUE encoding = rb_enc_from_encoding(enc);
1740 char *s = (char *)name;
1741 int haslower = 0, hasupper = 0, valid = 0;
1742
1743 if (ISDIGIT(*s)) return;
1744 if (ISUPPER(*s)) {
1745 hasupper = 1;
1746 while (*++s && (ISALNUM(*s) || *s == '_')) {
1747 if (ISLOWER(*s)) haslower = 1;
1748 }
1749 }
1750 if (!*s) {
1751 if (s - name > ENCODING_NAMELEN_MAX) return;
1752 valid = 1;
1753 rb_define_const(rb_cEncoding, name, encoding);
1754 }
1755 if (!valid || haslower) {
1756 size_t len = s - name;
1757 if (len > ENCODING_NAMELEN_MAX) return;
1758 if (!haslower || !hasupper) {
1759 do {
1760 if (ISLOWER(*s)) haslower = 1;
1761 if (ISUPPER(*s)) hasupper = 1;
1762 } while (*++s && (!haslower || !hasupper));
1763 len = s - name;
1764 }
1765 len += strlen(s);
1766 if (len++ > ENCODING_NAMELEN_MAX) return;
1767 MEMCPY(s = ALLOCA_N(char, len), name, char, len);
1768 name = s;
1769 if (!valid) {
1770 if (ISLOWER(*s)) *s = ONIGENC_ASCII_CODE_TO_UPPER_CASE((int)*s);
1771 for (; *s; ++s) {
1772 if (!ISALNUM(*s)) *s = '_';
1773 }
1774 if (hasupper) {
1775 rb_define_const(rb_cEncoding, name, encoding);
1776 }
1777 }
1778 if (haslower) {
1779 for (s = (char *)name; *s; ++s) {
1780 if (ISLOWER(*s)) *s = ONIGENC_ASCII_CODE_TO_UPPER_CASE((int)*s);
1781 }
1782 rb_define_const(rb_cEncoding, name, encoding);
1783 }
1784 }
1785}
1786
1787static int
1788rb_enc_name_list_i(st_data_t name, st_data_t idx, st_data_t arg)
1789{
1790 VALUE ary = (VALUE)arg;
1791 VALUE str = rb_fstring_cstr((char *)name);
1792 rb_ary_push(ary, str);
1793 return ST_CONTINUE;
1794}
1795
1796/*
1797 * call-seq:
1798 * Encoding.name_list -> ["enc1", "enc2", ...]
1799 *
1800 * Returns the list of available encoding names.
1801 *
1802 * Encoding.name_list
1803 * #=> ["US-ASCII", "ASCII-8BIT", "UTF-8",
1804 * "ISO-8859-1", "Shift_JIS", "EUC-JP",
1805 * "Windows-31J",
1806 * "BINARY", "CP932", "eucJP"]
1807 *
1808 */
1809
1810static VALUE
1811rb_enc_name_list(VALUE klass)
1812{
1813 VALUE ary = rb_ary_new2(global_enc_table.names->num_entries);
1814 st_foreach(global_enc_table.names, rb_enc_name_list_i, (st_data_t)ary);
1815 return ary;
1816}
1817
1818static int
1819rb_enc_aliases_enc_i(st_data_t name, st_data_t orig, st_data_t arg)
1820{
1821 VALUE *p = (VALUE *)arg;
1822 VALUE aliases = p[0], ary = p[1];
1823 int idx = (int)orig;
1824 VALUE key, str = rb_ary_entry(ary, idx);
1825
1826 if (NIL_P(str)) {
1827 rb_encoding *enc = rb_enc_from_index(idx);
1828
1829 if (!enc) return ST_CONTINUE;
1830 if (STRCASECMP((char*)name, rb_enc_name(enc)) == 0) {
1831 return ST_CONTINUE;
1832 }
1833 str = rb_fstring_cstr(rb_enc_name(enc));
1834 rb_ary_store(ary, idx, str);
1835 }
1836 key = rb_fstring_cstr((char *)name);
1837 rb_hash_aset(aliases, key, str);
1838 return ST_CONTINUE;
1839}
1840
1841/*
1842 * call-seq:
1843 * Encoding.aliases -> {"alias1" => "orig1", "alias2" => "orig2", ...}
1844 *
1845 * Returns the hash of available encoding alias and original encoding name.
1846 *
1847 * Encoding.aliases
1848 * #=> {"BINARY"=>"ASCII-8BIT", "ASCII"=>"US-ASCII", "ANSI_X3.4-1968"=>"US-ASCII",
1849 * "SJIS"=>"Windows-31J", "eucJP"=>"EUC-JP", "CP932"=>"Windows-31J"}
1850 *
1851 */
1852
1853static VALUE
1854rb_enc_aliases(VALUE klass)
1855{
1856 VALUE aliases[2];
1857 aliases[0] = rb_hash_new();
1858 aliases[1] = rb_ary_new();
1859
1860 st_foreach(global_enc_table.names, rb_enc_aliases_enc_i, (st_data_t)aliases);
1861
1862 return aliases[0];
1863}
1864
1865/*
1866 * An \Encoding instance represents a character encoding usable in Ruby.
1867 * It is defined as a constant under the \Encoding namespace.
1868 * It has a name and, optionally, aliases:
1869 *
1870 * Encoding::US_ASCII.name # => "US-ASCII"
1871 * Encoding::US_ASCII.names # => ["US-ASCII", "ASCII", "ANSI_X3.4-1968", "646"]
1872 *
1873 * A Ruby method that accepts an encoding as an argument will accept:
1874 *
1875 * - An \Encoding object.
1876 * - The name of an encoding.
1877 * - An alias for an encoding name.
1878 *
1879 * These are equivalent:
1880 *
1881 * 'foo'.encode(Encoding::US_ASCII) # Encoding object.
1882 * 'foo'.encode('US-ASCII') # Encoding name.
1883 * 'foo'.encode('ASCII') # Encoding alias.
1884 *
1885 * For a full discussion of encodings and their uses,
1886 * see {the Encodings document}[rdoc-ref:encodings.rdoc].
1887 *
1888 * Encoding::ASCII_8BIT is a special-purpose encoding that is usually used for
1889 * a string of bytes, not a string of characters.
1890 * But as the name indicates, its characters in the ASCII range
1891 * are considered as ASCII characters.
1892 * This is useful when you use other ASCII-compatible encodings.
1893 *
1894 */
1895
1896void
1897Init_Encoding(void)
1898{
1899 VALUE list;
1900 int i;
1901
1902 rb_cEncoding = rb_define_class("Encoding", rb_cObject);
1903 rb_define_alloc_func(rb_cEncoding, enc_s_alloc);
1905 rb_define_method(rb_cEncoding, "to_s", enc_name, 0);
1906 rb_define_method(rb_cEncoding, "inspect", enc_inspect, 0);
1907 rb_define_method(rb_cEncoding, "name", enc_name, 0);
1908 rb_define_method(rb_cEncoding, "names", enc_names, 0);
1909 rb_define_method(rb_cEncoding, "dummy?", enc_dummy_p, 0);
1910 rb_define_method(rb_cEncoding, "ascii_compatible?", enc_ascii_compatible_p, 0);
1911 rb_define_singleton_method(rb_cEncoding, "list", enc_list, 0);
1912 rb_define_singleton_method(rb_cEncoding, "name_list", rb_enc_name_list, 0);
1913 rb_define_singleton_method(rb_cEncoding, "aliases", rb_enc_aliases, 0);
1914 rb_define_singleton_method(rb_cEncoding, "find", enc_find, 1);
1915 rb_define_singleton_method(rb_cEncoding, "compatible?", enc_compatible_p, 2);
1916
1917 rb_define_method(rb_cEncoding, "_dump", enc_dump, -1);
1918 rb_define_singleton_method(rb_cEncoding, "_load", enc_load, 1);
1919
1920 rb_define_singleton_method(rb_cEncoding, "default_external", get_default_external, 0);
1921 rb_define_singleton_method(rb_cEncoding, "default_external=", set_default_external, 1);
1922 rb_define_singleton_method(rb_cEncoding, "default_internal", get_default_internal, 0);
1923 rb_define_singleton_method(rb_cEncoding, "default_internal=", set_default_internal, 1);
1924 rb_define_singleton_method(rb_cEncoding, "locale_charmap", rb_locale_charmap, 0); /* in localeinit.c */
1925
1926 struct enc_table *enc_table = &global_enc_table;
1927
1928 list = rb_encoding_list = rb_ary_new2(ENCODING_LIST_CAPA);
1929 RBASIC_CLEAR_CLASS(list);
1930 rb_gc_register_mark_object(list);
1931
1932 for (i = 0; i < enc_table->count; ++i) {
1933 rb_ary_push(list, enc_new(enc_table->list[i].enc));
1934 }
1935
1936 rb_marshal_define_compat(rb_cEncoding, Qnil, 0, enc_m_loader);
1937}
1938
1939void
1940Init_encodings(void)
1941{
1942 rb_enc_init(&global_enc_table);
1943}
1944
1945/* locale insensitive ctype functions */
1946
1947void
1948rb_enc_foreach_name(int (*func)(st_data_t name, st_data_t idx, st_data_t arg), st_data_t arg)
1949{
1950 st_foreach(global_enc_table.names, func, arg);
1951}
#define RUBY_ASSERT(expr)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:177
#define rb_define_method(klass, mid, func, arity)
Defines klass#mid.
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
static bool rb_enc_isascii(OnigCodePoint c, rb_encoding *enc)
Identical to rb_isascii(), except it additionally takes an encoding.
Definition ctype.h:82
VALUE rb_enc_sprintf(rb_encoding *enc, const char *fmt,...)
Identical to rb_sprintf(), except it additionally takes an encoding.
Definition sprintf.c:1200
@ RUBY_FL_SHAREABLE
This flag has something to do with Ractor.
Definition fl_type.h:266
VALUE rb_define_class(const char *name, VALUE super)
Defines a top-level class.
Definition class.c:970
void rb_undef_method(VALUE klass, const char *name)
Defines an undef of a method.
Definition class.c:2160
#define ENCODING_SET_INLINED(obj, i)
Old name of RB_ENCODING_SET_INLINED.
Definition encoding.h:105
#define ENC_CODERANGE_7BIT
Old name of RUBY_ENC_CODERANGE_7BIT.
Definition coderange.h:180
#define T_FILE
Old name of RUBY_T_FILE.
Definition value_type.h:62
#define T_STRING
Old name of RUBY_T_STRING.
Definition value_type.h:78
#define xfree
Old name of ruby_xfree.
Definition xmalloc.h:58
#define ISUPPER
Old name of rb_isupper.
Definition ctype.h:89
#define SPECIAL_CONST_P
Old name of RB_SPECIAL_CONST_P.
#define UNREACHABLE_RETURN
Old name of RBIMPL_UNREACHABLE_RETURN.
Definition assume.h:29
#define T_DATA
Old name of RUBY_T_DATA.
Definition value_type.h:60
#define CLASS_OF
Old name of rb_class_of.
Definition globals.h:203
#define xmalloc
Old name of ruby_xmalloc.
Definition xmalloc.h:53
#define ISDIGIT
Old name of rb_isdigit.
Definition ctype.h:93
#define ISLOWER
Old name of rb_islower.
Definition ctype.h:90
#define ASSUME
Old name of RBIMPL_ASSUME.
Definition assume.h:27
#define MBCLEN_CHARFOUND_LEN(ret)
Old name of ONIGENC_MBCLEN_CHARFOUND_LEN.
Definition encoding.h:516
#define ENCODING_INLINE_MAX
Old name of RUBY_ENCODING_INLINE_MAX.
Definition encoding.h:66
#define STRCASECMP
Old name of st_locale_insensitive_strcasecmp.
Definition ctype.h:102
#define ISASCII
Old name of rb_isascii.
Definition ctype.h:85
#define TOLOWER
Old name of rb_tolower.
Definition ctype.h:101
#define NUM2INT
Old name of RB_NUM2INT.
Definition int.h:44
#define INT2NUM
Old name of RB_INT2NUM.
Definition int.h:43
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define NIL_P
Old name of RB_NIL_P.
#define MBCLEN_CHARFOUND_P(ret)
Old name of ONIGENC_MBCLEN_CHARFOUND_P.
Definition encoding.h:515
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
Definition value_type.h:80
#define ENC_CODERANGE_ASCIIONLY(obj)
Old name of RB_ENC_CODERANGE_ASCIIONLY.
Definition coderange.h:185
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
Definition value_type.h:85
#define ENCODING_GET_INLINED(obj)
Old name of RB_ENCODING_GET_INLINED.
Definition encoding.h:107
#define ENC_CODERANGE_CLEAR(obj)
Old name of RB_ENC_CODERANGE_CLEAR.
Definition coderange.h:187
#define CONST_ID
Old name of RUBY_CONST_ID.
Definition symbol.h:47
#define rb_ary_new2
Old name of rb_ary_new_capa.
Definition array.h:651
#define ISALNUM
Old name of rb_isalnum.
Definition ctype.h:91
#define FL_SET_RAW
Old name of RB_FL_SET_RAW.
Definition fl_type.h:130
#define SYMBOL_P
Old name of RB_SYMBOL_P.
Definition value_type.h:88
#define T_REGEXP
Old name of RUBY_T_REGEXP.
Definition value_type.h:77
#define ruby_debug
This variable controls whether the interpreter is in debug mode.
Definition error.h:482
VALUE rb_eTypeError
TypeError exception.
Definition error.c:1344
VALUE rb_eEncCompatError
Encoding::CompatibilityError exception.
Definition error.c:1351
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:423
void rb_loaderror(const char *fmt,...)
Raises an instance of rb_eLoadError.
Definition error.c:3474
VALUE rb_eEncodingError
EncodingError exception.
Definition error.c:1350
void rb_warning(const char *fmt,...)
Issues a warning.
Definition error.c:454
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
Definition object.c:215
VALUE rb_cEncoding
Encoding class.
Definition encoding.c:57
Encoding relates APIs.
VALUE rb_locale_charmap(VALUE klass)
Returns a platform-depended "charmap" of the current locale.
Definition localeinit.c:91
static OnigCodePoint rb_enc_mbc_to_codepoint(const char *p, const char *e, rb_encoding *enc)
Identical to rb_enc_codepoint(), except it assumes the passed character is not broken.
Definition encoding.h:590
static int rb_enc_mbminlen(rb_encoding *enc)
Queries the minimum number of bytes that the passed encoding needs to represent a character.
Definition encoding.h:431
int rb_enc_str_coderange(VALUE str)
Scans the passed string to collect its code range.
Definition string.c:769
int rb_enc_str_asciionly_p(VALUE str)
Queries if the passed string is "ASCII only".
Definition string.c:781
#define rb_check_frozen
Just another name of rb_check_frozen
Definition error.h:264
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
Definition error.h:280
VALUE rb_check_string_type(VALUE obj)
Try converting an object to its stringised representation using its to_str method,...
Definition string.c:2686
VALUE rb_ivar_set(VALUE obj, ID name, VALUE val)
Identical to rb_iv_set(), except it accepts the name as an ID instead of a C string.
Definition variable.c:1854
VALUE rb_ivar_get(VALUE obj, ID name)
Identical to rb_iv_get(), except it accepts the name as an ID instead of a C string.
Definition variable.c:1340
void rb_define_alloc_func(VALUE klass, rb_alloc_func_t func)
Sets the allocator function of a class.
VALUE rb_sym2str(VALUE id)
Identical to rb_id2str(), except it takes an instance of rb_cSymbol rather than an ID.
Definition symbol.c:953
void rb_define_const(VALUE klass, const char *name, VALUE val)
Defines a Ruby level constant under a namespace.
Definition variable.c:3690
int len
Length of the buffer.
Definition io.h:8
#define strdup(s)
Just another name of ruby_strdup.
Definition util.h:187
void rb_marshal_define_compat(VALUE newclass, VALUE oldclass, VALUE(*dumper)(VALUE), VALUE(*loader)(VALUE, VALUE))
Marshal format compatibility layer.
Definition marshal.c:150
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition memory.h:366
#define ALLOCA_N(type, n)
Definition memory.h:286
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:161
#define DATA_PTR(obj)
Convenient getter macro.
Definition rdata.h:71
#define RDATA(obj)
Convenient casting macro.
Definition rdata.h:63
#define StringValue(v)
Ensures that the parameter object is a String.
Definition rstring.h:66
#define TypedData_Wrap_Struct(klass, data_type, sval)
Converts sval, a pointer to your struct, into a Ruby object.
Definition rtypeddata.h:449
This is the struct that holds necessary info for a struct.
Definition rtypeddata.h:200
Definition encoding.c:62
Definition st.h:79
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40