Fallback to utf-8 or iso-8859-1 if decoding using chardet-detected encoding fails

[vries]

Consider file gdb/testsuite/gdb.ada/non-ascii-utf-8/pack.adb from the binutils-gdb repo.

This works without problems:

$ codespell gdb/testsuite/gdb.ada/non-ascii-utf-8/pack.adb
$ 

which means utf-8 decoding is used.

However, with --hard-encoding-detection we get:

$ codespell --hard-encoding-detection gdb/testsuite/gdb.ada/non-ascii-utf-8/pack.adb
Traceback (most recent call last):
  File "/home/vries/codespell/python-3.9-venv/bin/codespell", line 6, in <module>
    sys.exit(_script_main())
  File "/home/vries/codespell/codespell.git/codespell_lib/_codespell.py", line 1294, in _script_main
    return main(*sys.argv[1:])
  File "/home/vries/codespell/codespell.git/codespell_lib/_codespell.py", line 1534, in main
    bad_count += parse_file(
  File "/home/vries/codespell/codespell.git/codespell_lib/_codespell.py", line 1217, in parse_file
    fragments, encoding = file_opener.open(filename)
  File "/home/vries/codespell/codespell.git/codespell_lib/_codespell.py", line 250, in open
    return self.open_with_chardet(filename)
  File "/home/vries/codespell/codespell.git/codespell_lib/_codespell.py", line 277, in open_with_chardet
    lines = self.get_lines(f)
  File "/home/vries/codespell/codespell.git/codespell_lib/_codespell.py", line 328, in get_lines
    fragments.append((False, line_number, f.readlines()))
  File "/usr/lib64/python3.9/encodings/cp1254.py", line 23, in decode
UnicodeDecodeError: 'charmap' codec can't decode byte 0x90 in position 943: character maps to <undefined>

What happens is that chardet detects CP1254 (aka Windows-1254) encoding, but decoding using that encoding fails.

When not using --hard-encoding-detection, we first try utf-8, and if that doesn't work, warn and try iso-8859-1.

It seems logical to extend the fallback handling to --hard-encoding-detection:

$ codespell --hard-encoding-detection gdb/testsuite/gdb.ada/non-ascii-utf-8/pack.adb
WARNING: Cannot decode file using encoding "Windows-1254": gdb/testsuite/gdb.ada/non-ascii-utf-8/pack.adb
WARNING: Trying next encoding "utf-8"
$

[vries]

Tentative patch:

diff --git a/codespell_lib/_codespell.py b/codespell_lib/_codespell.py
index 994c5c4a..bb462963 100644
--- a/codespell_lib/_codespell.py
+++ b/codespell_lib/_codespell.py
@@ -245,10 +245,23 @@ class FileOpener:
 
         self.encdetector = UniversalDetector()
 
+    def warn_decode(self, filename: str, encoding: str) -> None:
+        if not self.quiet_level & QuietLevels.ENCODING:
+            print(
+                f'WARNING: Cannot decode file using encoding "{encoding}": '
+                f"{filename}",
+                file=sys.stderr,
+            )
+
     def open(self, filename: str) -> tuple[list[tuple[bool, int, list[str]]], str]:
-        if self.use_chardet:
-            return self.open_with_chardet(filename)
-        return self.open_with_internal(filename)
+        first_try = True
+        try:
+            if self.use_chardet:
+                return self.open_with_chardet(filename)
+        except UnicodeDecodeError:
+            first_try = False
+            self.warn_decode(filename, self.encoding)
+        return self.open_with_internal(filename, first_try)
 
     def open_with_chardet(
         self, filename: str
@@ -261,6 +274,7 @@ class FileOpener:
                     break
         self.encdetector.close()
         encoding = self.encdetector.result["encoding"]
+        self.encoding = encoding
 
         try:
             f = open(filename, encoding=encoding, newline="")
@@ -280,10 +294,9 @@ class FileOpener:
         return lines, f.encoding
 
     def open_with_internal(
-        self, filename: str
+        self, filename: str, first_try: bool
     ) -> tuple[list[tuple[bool, int, list[str]]], str]:
         encoding = None
-        first_try = True
         for encoding in ("utf-8", "iso-8859-1"):
             if first_try:
                 first_try = False
@@ -293,12 +306,7 @@ class FileOpener:
                 try:
                     lines = self.get_lines(f)
                 except UnicodeDecodeError:
-                    if not self.quiet_level & QuietLevels.ENCODING:
-                        print(
-                            f'WARNING: Cannot decode file using encoding "{encoding}": '
-                            f"{filename}",
-                            file=sys.stderr,
-                        )
+                    self.warn_decode(filename, encoding)
                 else:
                     break
         else: