character encodings

# a text like this one

text1 = "abcd\nefgh\n"

# and to confirm that, we can use ord()
# which applies on a single char (raises an exception otherwise)

for c in text1:
    print(f"{c} -> {ord(c)}")

# FYI, note we also have the reverse function, pour info, qu'on a aussi la fonction inverse de ord
# qui s'appelle chr()

chr(97)

from pathlib import Path

# remember to always use a with: when dealing with files

with Path('encodings1-utf8').open('w', encoding='UTF-8') as f:
    f.write(text1)

text2 = "abçd\néfgh\n"

# notice the values > 127
# which are not supported in ASCII

for c in text2:
    print(f"{c} -> {ord(c)}")

# focus on the 2 characters whose encoding is > 127

hex(231), hex(233)

bin(231), bin(233)

chr(231), ord('ç')

with Path('encodings2-utf8').open('w', encoding='UTF-8') as f:
    f.write(text2)

# no encoding in binary mode, would make no sense !

with Path('encodings2-utf8').open('rb') as f:
    raw = f.read()

len(raw), len(text2)

# here is it again
raw

# extract the 2-bytes areas for each alien character

ccedilla = raw[2:4]
eaccent = raw[6:8]

for b in ccedilla:
    print(f"byte {b} {hex(b)} {bin(b)}")

for b in eaccent:
    print(f"byte {b} {hex(b)} {bin(b)}")

# we want 5 bits from the first byte and 6 from the second byte

on2bytes_0_len = 5
on2bytes_1_len = 6

# and that's what should occur in the remaining (left-hand-side) bits
on2bytes_0_pad = 0b110
on2bytes_1_pad = 0b10

def mask_from_len(length):
    """
    for e.g. len == 5, we compute a mask that has
    3 bits set and 5 bits unset (because 3+5=8)
    """
    return 2**8 - 2**length

# let us check that it works as advertised:

# e.g. for byte0
# the result allows to separate 
# the (3-bits) padding from 
# the (5-bits) payload

bin(mask_from_len(5))

# with that we can manually decode 2-bytes UTF-8 !

on2bytes_0_mask = mask_from_len(on2bytes_0_len)
on2bytes_1_mask = mask_from_len(on2bytes_1_len)

def decode(on2bytes):
    b0, b1 = on2bytes
    # check masks
    # e.g. check that the 3 high bits in 0xc9 are indeed 0b110
    assert (b0 & on2bytes_0_mask) >> on2bytes_0_len == on2bytes_0_pad
    # same on byte 1
    assert (b1 & on2bytes_1_mask) >> on2bytes_1_len == on2bytes_1_pad
    # extract meaningful bits
    # for that we just need to invert the mask
    bits0 = b0 & ~ (on2bytes_0_mask)
    bits1 = b1 & ~ (on2bytes_1_mask)
    # asemble bits into codepoint
    # b0 has the high bits so it needs to be shifted
    # by the number of meaningful bits in byte1
    codepoint = bits1 | bits0 << on2bytes_1_len
    return chr(codepoint)

# and indeed 
decode(eaccent), decode(ccedilla)

with Path("encodings2-utf32").open('w', encoding='utf-32') as f:
    f.write(text2)

# computing file size: use pathlib !

p = Path("encodings2-utf32")
print(f"file has {p.stat().st_size} bytes")

len(text2)

# read the 4 first bytes

with Path("encodings2-utf32").open('rb') as f:
    bom = f.read(4)

bom

# read the whole file

with Path("encodings2-utf32").open('rb') as f:
    raw = f.read()

# remember that ç is at index 2

index = 2

text2[index]

offset = 4 + 4*index

b4 = raw[offset:offset+4]

b4

# int.from_bytes knowns how to transform a sequence
# of bytes into an int, given the endian-ness

int.from_bytes(b4, 'little')

# and indeed, that is what was encoded in the file !

chr(231)