From 09c621bf8d8e527422b4c40dcf595b122bee122d Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 30 Jul 2026 15:58:11 +0300 Subject: [PATCH 1/2] mofile: decode context (and optimize with partition) --- babel/messages/mofile.py | 17 ++++++++++------- tests/messages/test_mofile.py | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/babel/messages/mofile.py b/babel/messages/mofile.py index 1a6fedfcb..3f5317664 100644 --- a/babel/messages/mofile.py +++ b/babel/messages/mofile.py @@ -82,16 +82,19 @@ def read_mo(fileobj: SupportsRead[bytes]) -> Catalog: elif lastkey: headers[lastkey] += b'\n' + item - if b'\x04' in msg: # context - ctxt, msg = msg.split(b'\x04') + ctxt = None + # > Contexts are stored by storing the concatenation of the context, + # > a EOT byte, and the original string, instead of the original string. + # - https://www.gnu.org/software/gettext/manual/html_node/MO-Files.html + ctxt_or_msg, sep, msg = msg.partition(b'\x04') + if sep: + ctxt = ctxt_or_msg.decode(catalog.charset) else: - ctxt = None + msg = ctxt_or_msg if b'\x00' in msg: # plural forms - msg = msg.split(b'\x00') - tmsg = tmsg.split(b'\x00') - msg = [x.decode(catalog.charset) for x in msg] - tmsg = [x.decode(catalog.charset) for x in tmsg] + msg = [x.decode(catalog.charset) for x in msg.split(b'\x00')] + tmsg = [x.decode(catalog.charset) for x in tmsg.split(b'\x00')] else: msg = msg.decode(catalog.charset) tmsg = tmsg.decode(catalog.charset) diff --git a/tests/messages/test_mofile.py b/tests/messages/test_mofile.py index 85f4e9f34..a96f4d830 100644 --- a/tests/messages/test_mofile.py +++ b/tests/messages/test_mofile.py @@ -84,3 +84,23 @@ def test_empty_translation_with_fallback(): translations.add_fallback(Translations(fp=buf2)) assert translations.ugettext('Fuzz') == 'Flou' + + +def test_read_mo_decodes_message_context() -> None: + catalog = Catalog(locale='fi_FI') + catalog.add('', '''\ +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n''') + catalog.add('x', 'åäöÅÄÖ', context='ほげ') + buf = BytesIO() + mofile.write_mo(buf, catalog) + buf.seek(0) + del catalog + + catalog = mofile.read_mo(buf) + message = catalog.get('x', context='ほげ') + assert message is not None + assert message.id == 'x' + assert message.context == 'ほげ' + assert message.string == 'åäöÅÄÖ' + assert not catalog.get('x') # Not without the context From d55968149db7adf3a90c0f21f612210fae2a8e1f Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 30 Jul 2026 16:06:51 +0300 Subject: [PATCH 2/2] mofile: Separate byte data from decoded data --- babel/messages/mofile.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/babel/messages/mofile.py b/babel/messages/mofile.py index 3f5317664..2413f7139 100644 --- a/babel/messages/mofile.py +++ b/babel/messages/mofile.py @@ -62,8 +62,8 @@ def read_mo(fileobj: SupportsRead[bytes]) -> Catalog: tlen, toff = unpack(ii, buf[transidx : transidx + 8]) tend = toff + tlen if mend < buflen and tend < buflen: - msg = buf[moff:mend] - tmsg = buf[toff:tend] + raw_msg: bytes = buf[moff:mend] + raw_tmsg: bytes = buf[toff:tend] else: raise OSError(0, 'File is corrupt', filename) @@ -71,7 +71,7 @@ def read_mo(fileobj: SupportsRead[bytes]) -> Catalog: if mlen == 0: # Catalog description lastkey = key = None - for item in tmsg.splitlines(): + for item in raw_tmsg.splitlines(): item = item.strip() if not item: continue @@ -82,22 +82,25 @@ def read_mo(fileobj: SupportsRead[bytes]) -> Catalog: elif lastkey: headers[lastkey] += b'\n' + item - ctxt = None + ctxt: str | None = None + msg: list[str] | str + tmsg: list[str] | str + # > Contexts are stored by storing the concatenation of the context, # > a EOT byte, and the original string, instead of the original string. # - https://www.gnu.org/software/gettext/manual/html_node/MO-Files.html - ctxt_or_msg, sep, msg = msg.partition(b'\x04') + ctxt_or_msg, sep, raw_msg = raw_msg.partition(b'\x04') if sep: ctxt = ctxt_or_msg.decode(catalog.charset) else: - msg = ctxt_or_msg + raw_msg = ctxt_or_msg - if b'\x00' in msg: # plural forms - msg = [x.decode(catalog.charset) for x in msg.split(b'\x00')] - tmsg = [x.decode(catalog.charset) for x in tmsg.split(b'\x00')] + if b'\x00' in raw_msg: # plural forms + msg = [x.decode(catalog.charset) for x in raw_msg.split(b'\x00')] + tmsg = [x.decode(catalog.charset) for x in raw_tmsg.split(b'\x00')] else: - msg = msg.decode(catalog.charset) - tmsg = tmsg.decode(catalog.charset) + msg = raw_msg.decode(catalog.charset) + tmsg = raw_tmsg.decode(catalog.charset) catalog[msg] = Message(msg, tmsg, context=ctxt) # advance to next entry in the seek tables