Friendsnippets

1. unicode2ascii() : conversion tout Unicode vers ASCII

   view plain - save this

   Convertit n'importe quelle chaîne de caractères Unicode en chaîne de caractère ASCII : supprime les accents et remplace les lettres spéciales par des caractères ASCII semblables.

     1 #!/usr/bin/python
     2 # -*- coding: utf-8 -*-
     3 
     4 # This source code is distributed under GNU GPL v2 license
     5 # written by Victor Stinner <victor.stinner AT haypocalc.com>
     6 #                            http://www.haypocalc.com/
     7 # creatied: 2006-08-14 -- last change: 2007-08-17
     8 
     9 # Convert any unicode string to ASCII string:
    10 #  - Remove diacriticals
    11 #  - Replace special letter with similar ASCII character (similar glyph)
    12 #
    13 # Support greek, cyrillic, some latin letters and some signs.
    14 
    15 from unicodedata import normalize
    16 
    17 UNICODE_TO_ASCII = {
    18     # Latin letters
    19     u"Æ": u"AE",   # U+00C6 (latin capital ligature ae)
    20     u"Ø": u"O",    # U+00D8 (latin capital letter o with stroke)
    21     u"ß": u"ss",   # U+00DF (latin small letter sharp s)
    22     u"æ": u"ae",   # U+00E6 (latin small ligature ae)
    23     u"ø": u"o",    # U+00F8 (latin small letter o with stroke)
    24     u"?": u"l",    # U+0142 (latin small letter l with stroke)
    25     u"Œ": u"OE",   # U+0152 (latin capital ligature oe)
    26     u"œ": u"oe",   # U+0153 (latin small ligature oe)
    27 
    28     # Various signs
    29     u"¡": u"!",    # U+00A1 (inverted exclamation mark)
    30     u"©": u"(c)",  # U+00A9 (copyright sign)
    31     u"«": u'"',    # U+00AB (left-pointing double angle quotation mark)
    32     u"®": u"(r)",  # U+00AE (registred sign)
    33     u"²": u"2",    # U+00B2 (superscript two)
    34     u"»": u'"',    # U+00BB (right-pointing double angle quotation mark)
    35     u"?": u"/",    # U+2044 (fraction slash)
    36 
    37     # Greek
    38     u"?": u"A",    # U+0391 (capital alpha)
    39     u"?": u"B",    # U+0392 (capital beta)
    40     u"?": u"E",    # U+0395 (capital epsilon)
    41     u"?": u"Z",    # U+0396 (capital zeta)
    42     u"?": u"H",    # U+0397 (capital eta)
    43     u"?": u"O",    # U+0398 (captial theta)
    44     u"?": u"I",    # U+0399 (capital iota)
    45     u"?": u"K",    # U+039A (capital kappa)
    46     u"?": u"M",    # U+039C (capital mu)
    47     u"?": u"N",    # U+039D (capital nu)
    48     u"?": u"O",    # U+039F (capital omicron)
    49     u"?": u"P",    # U+03A1 (capital rho)
    50     u"?": u"T",    # U+03A4 (capital tau)
    51     u"?": u"Y",    # U+03A5 (capital upsilon)
    52     u"?": u"X",    # U+03A7 (capital chi)
    53     u"?": u"a",    # U+03B1 (small alpha)
    54     u"?": u"b",    # U+03B2 (small beta)
    55     u"?": u"y",    # U+03B2 (small gamma)
    56     u"?": u"e",    # U+03B5 (small espilon)
    57     u"?": u"n",    # U+03B7 (small eta)
    58     u"?": u"o",    # U+03BF (small omicron)
    59     u"?": u"p",    # U+03C1 (small rho)
    60     u"?": u"v",    # U+03C1 (small upsilon)
    61 
    62     # Cyrillic
    63     u"?": u"I",    # U+0406 (capital byelorussian-ukrainian i)
    64     u"?": u"J",    # U+0408 (capital je)
    65     u"?": u"B",    # U+0412 (capital ve)
    66     u"?": u"E",    # U+0415 (capital ie)
    67     u"?": u"N",    # U+0418 (capital i)
    68     u"?": u"3",    # U+0417 (capital ze)
    69     u"?": u"K",    # U+041A (capital ka)
    70     u"?": u"M",    # U+041C (capital em)
    71     u"?": u"H",    # U+041D (capital en)
    72     u"?": u"O",    # U+041E (capital o)
    73     u"?": u"P",    # U+0420 (capital er)
    74     u"?": u"C",    # U+0421 (capital es)
    75     u"?": u"T",    # U+0422 (capital te)
    76     u"?": u"Y",    # U+0423 (capital u)
    77     u"?": u"X",    # U+0425 (capital ha)
    78     u"?": u"R",    # U+042F (capital ya)
    79     u"?": u"a",    # U+0430 (small a)
    80     u"?": u"b",    # U+0432 (small ve)
    81     u"?": u"e",    # U+0435 (small ie)
    82     u"?": u"3",    # U+0437 (small ze)
    83     u"?": u"k",    # U+043A (small ka)
    84     u"?": u"m",    # U+043C (small em)
    85     u"?": u"h",    # U+043D (small en)
    86     u"?": u"o",    # U+043E (small o)
    87     u"?": u"p",    # U+0440 (small er)
    88     u"?": u"c",    # U+0441 (small es)
    89     u"?": u"T",    # U+0442 (small te)
    90     u"?": u"y",    # U+0443 (small u)
    91     u"?": u"x",    # U+0445 (small ha)
    92     u"?": u"R",    # U+044F (small ya)
    93     u"?": u"i",    # U+0456 (small byelorussian-ukrainian i)
    94     u"?": u"j",    # U+0458 (small je)
    95 }
    96 
    97 def unicode2ascii(text, replace=False):
    98     """
    99     Convert an unicode string (type 'unicode') to ascii string (type 'str').
   100     Try to keep same visual result.
   101 
   102     You can specify an ASCII character to replace non-ASCII character
   103     in 'replace' argument (eg. replace='?').
   104 
   105     >>> unicode2ascii(unicode("¡ Hé hø « español » ! Pythøn", "UTF-8"))
   106     '! He ho " espanol " ! Python'
   107     >>> unicode2ascii(unicode("L'œuf de læticia", "UTF-8"))
   108     "L'oeuf de laeticia"
   109     >>> unicode2ascii(unicode("????????????????????????????", "UTF-8"), u'?')
   110     'IEOAB??EZHOIK?NM?OYanay?e?n?'
   111     >>> unicode2ascii(unicode("??????????????????????????????????????????", "UTF-8"), u'?')
   112     'EE??IIJKN?BE3NKMHOPCTYXabe3mho?pcTyxeeiijk'
   113     """
   114     assert isinstance(text, unicode)
   115     if replace:
   116         if isinstance(replace, str):
   117             replace = unicode(replace, "latin-1")
   118         if not isinstance(replace, unicode) \
   119         or len(replace) != 1 \
   120         or not (32 <= ord(replace) <= 127):
   121             raise ValueError(
   122                 "invalid replace character (%r): "
   123                 "need one ascii printable character" % replace)
   124 
   125     ascii = []
   126     for char in text:
   127         # Remove diacriticals
   128         char = normalize("NFKD", char)[0]
   129 
   130         # Known values
   131         if char in UNICODE_TO_ASCII:
   132             ascii.append(UNICODE_TO_ASCII[char])
   133             continue
   134 
   135         if ord(char) <= 127:
   136             # Add valid ASCII
   137             ascii.append(char)
   138         elif replace:
   139             # non-ASCII character
   140             ascii.append(replace)
   141         # else: ignore it
   142 
   143     text = ''.join(ascii)
   144     return text.encode("ascii", "strict")
   145 
   146 if __name__ == "__main__":
   147     from doctest import testmod
   148     from sys import exit
   149     failure, total = testmod()
   150     if failure:
   151         print "%s failure on %s tests" % (failure, total)
   152         exit(1)
   153     else:
   154         print "All tests are OK (count=%s)" % total
   

   Posted by haypo to python unicode ascii accents characters ... saved by 7 persons ... 2 comments ... 1 year, 1 month

2. Remplace les caractères accentués par leurs équivalents

   view plain - save this

   Exemple d'utilisation Accents.clean("mämé") == "mame"

    1 class Accents:
    2     __reptable = {}
    3 
    4     __corresp = [
    5         (u"A",  [0x00C0,0x00C1,0x00C2,0x00C3,0x00C4,0x00C5,0x0100,0x0102,0x0104]),
    6         (u"AE", [0x00C6]),
    7         (u"a",  [0x00E0,0x00E1,0x00E2,0x00E3,0x00E4,0x00E5,0x0101,0x0103,0x0105]),
    8         (u"ae", [0x00E6]),
    9         (u"C",  [0x00C7,0x0106,0x0108,0x010A,0x010C]),
   10         (u"c",  [0x00E7,0x0107,0x0109,0x010B,0x010D]),
   11         (u"D",  [0x00D0,0x010E,0x0110]),
   12         (u"d",  [0x00F0,0x010F,0x0111]),
   13         (u"E",  [0x00C8,0x00C9,0x00CA,0x00CB,0x0112,0x0114,0x0116,0x0118,0x011A]),
   14         (u"e",  [0x00E8,0x00E9,0x00EA,0x00EB,0x0113,0x0115,0x0117,0x0119,0x011B]),
   15         (u"G",  [0x011C,0x011E,0x0120,0x0122]),
   16         (u"g",  [0x011D,0x011F,0x0121,0x0123]),
   17         (u"H",  [0x0124,0x0126]),
   18         (u"h",  [0x0125,0x0127]),
   19         (u"I",  [0x00CC,0x00CD,0x00CE,0x00CF,0x0128,0x012A,0x012C,0x012E,0x0130]),
   20         (u"i",  [0x00EC,0x00ED,0x00EE,0x00EF,0x0129,0x012B,0x012D,0x012F,0x0131]),
   21         (u"IJ", [0x0132]),
   22         (u"ij", [0x0133]),
   23         (u"J",  [0x0134]),
   24         (u"j",  [0x0135]),
   25         (u"K",  [0x0136]),
   26         (u"k",  [0x0137,0x0138]),
   27         (u"L",  [0x0139,0x013B,0x013D,0x013F,0x0141]),
   28         (u"l",  [0x013A,0x013C,0x013E,0x0140,0x0142]),
   29         (u"N",  [0x00D1,0x0143,0x0145,0x0147,0x014A]),
   30         (u"n",  [0x00F1,0x0144,0x0146,0x0148,0x0149,0x014B]),
   31         (u"O",  [0x00D2,0x00D3,0x00D4,0x00D5,0x00D6,0x00D8,0x014C,0x014E,0x0150]),
   32         (u"o",  [0x00F2,0x00F3,0x00F4,0x00F5,0x00F6,0x00F8,0x014D,0x014F,0x0151]),
   33         (u"OE", [0x0152]),
   34         (u"oe", [0x0153]),
   35         (u"R",  [0x0154,0x0156,0x0158]),
   36         (u"r",  [0x0155,0x0157,0x0159]),
   37         (u"S",  [0x015A,0x015C,0x015E,0x0160]),
   38         (u"s",  [0x015B,0x015D,0x015F,0x01610,0x017F]),
   39         (u"T",  [0x0162,0x0164,0x0166]),
   40         (u"t",  [0x0163,0x0165,0x0167]),
   41         (u"U",  [0x00D9,0x00DA,0x00DB,0x00DC,0x0168,0x016A,0x016C,0x016E,0x0170,0x172]),
   42         (u"u",  [0x00F9,0x00FA,0x00FB,0x00FC,0x0169,0x016B,0x016D,0x016F,0x0171]),
   43         (u"W",  [0x0174]),
   44         (u"w",  [0x0175]),
   45         (u"Y",  [0x00DD,0x0176,0x0178]),
   46         (u"y",  [0x00FD,0x00FF,0x0177]),
   47         (u"Z",  [0x0179,0x017B,0x017D]),
   48         (u"z",  [0x017A,0x017C,0x017E])
   49         ]
   50     for __repchar,__codes in __corresp :
   51         for __code in __codes :
   52             __reptable[__code] = __repchar
   53 
   54     @staticmethod
   55     def clean(s):
   56         """ enleve les accents de 's' (unicode ou utf8), et renvoi de l'unicode
   57             (marche pour maj et minuscules !!!)
   58         """
   59         if isinstance(s,str) :
   60             s = unicode(s,"utf8","replace")
   61         return s.translate(Accents.__reptable)
   

   Posted by manatlan to python characters accents diacritics ... saved by 11 persons ... 2 comments ... 1 year, 1 month

Pages : 1