sistema_progs

Programas para customizar o meu entorno de traballo nos meus equipos persoais
Log | Files | Refs

utf8.h (2336B)


      1 /* Branchless UTF-8 decoder
      2  *
      3  * This is free and unencumbered software released into the public domain.
      4  */
      5 #ifndef UTF8_H
      6 #define UTF8_H
      7 
      8 #include <stdint.h>
      9 
     10 /* Decode the next character, C, from BUF, reporting errors in E.
     11  *
     12  * Since this is a branchless decoder, four bytes will be read from the
     13  * buffer regardless of the actual length of the next character. This
     14  * means the buffer _must_ have at least three bytes of zero padding
     15  * following the end of the data stream.
     16  *
     17  * Errors are reported in E, which will be non-zero if the parsed
     18  * character was somehow invalid: invalid byte sequence, non-canonical
     19  * encoding, or a surrogate half.
     20  *
     21  * The function returns a pointer to the next character. When an error
     22  * occurs, this pointer will be a guess that depends on the particular
     23  * error, but it will always advance at least one byte.
     24  */
     25 static void *
     26 utf8_decode(void *buf, uint32_t *c, int *e)
     27 {
     28     static const char lengths[] = {
     29         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
     30         0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 3, 3, 4, 0
     31     };
     32     static const int masks[]  = {0x00, 0x7f, 0x1f, 0x0f, 0x07};
     33     static const uint32_t mins[] = {4194304, 0, 128, 2048, 65536};
     34     static const int shiftc[] = {0, 18, 12, 6, 0};
     35     static const int shifte[] = {0, 6, 4, 2, 0};
     36 
     37     unsigned char *s = buf;
     38     int len = lengths[s[0] >> 3];
     39 
     40     /* Compute the pointer to the next character early so that the next
     41      * iteration can start working on the next character. Neither Clang
     42      * nor GCC figure out this reordering on their own.
     43      */
     44     unsigned char *next = s + len + !len;
     45 
     46     /* Assume a four-byte character and load four bytes. Unused bits are
     47      * shifted out.
     48      */
     49     *c  = (uint32_t)(s[0] & masks[len]) << 18;
     50     *c |= (uint32_t)(s[1] & 0x3f) << 12;
     51     *c |= (uint32_t)(s[2] & 0x3f) <<  6;
     52     *c |= (uint32_t)(s[3] & 0x3f) <<  0;
     53     *c >>= shiftc[len];
     54 
     55     /* Accumulate the various error conditions. */
     56     *e  = (*c < mins[len]) << 6; // non-canonical encoding
     57     *e |= ((*c >> 11) == 0x1b) << 7;  // surrogate half?
     58     *e |= (*c > 0x10FFFF) << 8;  // out of range?
     59     *e |= (s[1] & 0xc0) >> 2;
     60     *e |= (s[2] & 0xc0) >> 4;
     61     *e |= (s[3]       ) >> 6;
     62     *e ^= 0x2a; // top two bits of each tail byte correct?
     63     *e >>= shifte[len];
     64 
     65     return next;
     66 }
     67 
     68 #endif