Mercurial > jython
changeset 8313:3e46a80390fb
Tolerate undecodable bytes sys.path elements (fixes #2820).
We treat these as import failures, and walk on down the sys.path.
author | Jeff Allen <ja.py@farowl.co.uk> |
---|---|
date | Sat, 21 Dec 2019 19:52:20 +0000 |
parents | 84ad2ab25bca |
children | 069db4761b37 |
files | NEWS src/org/python/core/PyNullImporter.java src/org/python/core/SyspathJavaLoader.java src/org/python/core/imp.java src/org/python/core/packagecache/PathPackageManager.java src/org/python/modules/zipimport/zipimporter.java |
diffstat | 6 files changed, 77 insertions(+), 34 deletions(-) [+] |
line wrap: on
line diff
--- a/NEWS +++ b/NEWS @@ -8,6 +8,7 @@ https://github.com/jythontools/jython Jython 2.7.2b3 Bugs fixed + - [ 2820 ] Import fails with UnicodeDecodeError if sys.path contains invalid UTF-8 bytes Jython 2.7.2b2 Bugs fixed
--- a/src/org/python/core/PyNullImporter.java +++ b/src/org/python/core/PyNullImporter.java @@ -20,11 +20,10 @@ public class PyNullImporter extends PyOb public PyNullImporter(PyObject pathObj) { super(); - String pathStr = Py.fileSystemDecode(pathObj); + String pathStr = imp.fileSystemDecode(pathObj); if (pathStr.equals("")) { throw Py.ImportError("empty pathname"); - } - if (isDir(pathStr)) { + } else if (isDir(pathStr)) { throw Py.ImportError("existing directory: " + pathStr); } }
--- a/src/org/python/core/SyspathJavaLoader.java +++ b/src/org/python/core/SyspathJavaLoader.java @@ -113,11 +113,11 @@ public class SyspathJavaLoader extends C byte[] buffer; PyObject entry = replacePathItem(sys, i, path); if (entry instanceof SyspathArchive) { - SyspathArchive archive = (SyspathArchive)entry; + SyspathArchive archive = (SyspathArchive) entry; buffer = getBytesFromArchive(archive, name); } else { - String dir = Py.fileSystemDecode(entry); - buffer = getBytesFromDir(dir, name); + String dir = imp.fileSystemDecode(entry, false); + buffer = dir != null ? getBytesFromDir(dir, name) : null; } if (buffer != null) { definePackageForClass(name);
--- a/src/org/python/core/imp.java +++ b/src/org/python/core/imp.java @@ -1,10 +1,6 @@ // Copyright (c) Corporation for National Research Initiatives package org.python.core; -import org.python.compiler.Module; -import org.python.core.util.FileUtil; -import org.python.core.util.PlatformUtil; - import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; @@ -13,8 +9,12 @@ import java.io.IOException; import java.io.InputStream; import java.util.Map; import java.util.concurrent.locks.ReentrantLock; +import java.util.logging.Level; import java.util.logging.Logger; -import java.util.logging.Level; + +import org.python.compiler.Module; +import org.python.core.util.FileUtil; +import org.python.core.util.PlatformUtil; /** * Utility functions for "import" support. @@ -744,7 +744,7 @@ public class imp { // Note the path here may be sys.path or the search path of a Python package. path = path == null ? sys.path : path; - for (int i = 0; i < path.__len__(); i++) { + for (int i = 0; ret == null && i < path.__len__(); i++) { PyObject p = path.__getitem__(i); // Is there a path-specific importer? PyObject importer = getPathImporter(sys.path_importer_cache, sys.path_hooks, p); @@ -757,9 +757,9 @@ public class imp { } } // p could be a unicode or bytes object (in the file system encoding) - ret = loadFromSource(sys, name, moduleName, Py.fileSystemDecode(p)); - if (ret != null) { - return ret; + String pathElement = fileSystemDecode(p, false); + if (pathElement != null) { + ret = loadFromSource(sys, name, moduleName, pathElement); } } @@ -1388,6 +1388,45 @@ public class imp { } /** + * A wrapper for {@link Py#fileSystemDecode(PyObject)} for <b>project internal use</b> within + * the import mechanism to convert decoding errors that occur during import to either + * {@code null} or {@link Py#ImportError(String)} calls (and a log message), which usually + * results in quiet failure. + * + * @param p assumed to be a (partial) file path + * @param raiseImportError if true and {@code p} cannot be decoded raise {@code ImportError}. + * @return String form of the object {@code p} (or {@code null}). + */ + public static String fileSystemDecode(PyObject p, boolean raiseImportError) { + try { + return Py.fileSystemDecode(p); + } catch (PyException e) { + if (e.match(Py.UnicodeDecodeError)) { + // p is bytes we cannot convert to a String using the FS encoding + if (raiseImportError) { + logger.log(Level.CONFIG, "Cannot decode path entry {0}", p.__repr__()); + throw Py.ImportError("cannot decode"); + } + return null; + } else { + // Any other kind of exception continues as itself + throw e; + } + } + } + + /** + * For <b>project internal use</b>, equivalent to {@code fileSystemDecode(p, true)} (see + * {@link #fileSystemDecode(PyObject, boolean)}). + * + * @param p assumed to be a (partial) file path + * @return String form of the object {@code p}. + */ + public static String fileSystemDecode(PyObject p) { + return fileSystemDecode(p, true); + } + + /** * Ensure that the items mentioned in the from-list of an import are actually present, even if * they are modules we have not imported yet. *
--- a/src/org/python/core/packagecache/PathPackageManager.java +++ b/src/org/python/core/packagecache/PathPackageManager.java @@ -11,6 +11,7 @@ import java.io.IOException; import java.util.logging.Level; import org.python.core.Py; +import org.python.core.PyException; import org.python.core.PyJavaPackage; import org.python.core.PyList; import org.python.core.PyObject; @@ -41,28 +42,30 @@ public abstract class PathPackageManager for (int i = 0; i < path.__len__(); i++) { - // Each entry in the path may be byte-encoded or unicode PyObject entry = path.pyget(i); - String dir = Py.fileSystemDecode(entry); - File f = new RelativeFile(dir, child); - try { - if (f.isDirectory() && imp.caseok(f, name)) { - /* - * f is a directory matching the package name. This directory is considered to - * define a package if it contains no Python (source or compiled), or contains a - * Java .class file (not compiled from Python). - */ - PackageExistsFileFilter m = new PackageExistsFileFilter(); - f.listFiles(m); - boolean exists = m.packageExists(); - if (exists) { - logger.log(Level.CONFIG, "# trying {0}", f.getAbsolutePath()); + // Each entry in the path may be byte-encoded or unicode + String dir = imp.fileSystemDecode(entry, false); + if (dir != null) { + File f = new RelativeFile(dir, child); + try { + if (f.isDirectory() && imp.caseok(f, name)) { + /* + * f is a directory matching the package name. This directory is considered + * to define a package if it contains no Python (source or compiled), or + * contains a Java .class file (not compiled from Python). + */ + PackageExistsFileFilter m = new PackageExistsFileFilter(); + f.listFiles(m); + boolean exists = m.packageExists(); + if (exists) { + logger.log(Level.CONFIG, "# trying {0}", f.getAbsolutePath()); + } + return exists; } - return exists; + } catch (SecurityException se) { + return false; } - } catch (SecurityException se) { - return false; } } return false;
--- a/src/org/python/modules/zipimport/zipimporter.java +++ b/src/org/python/modules/zipimport/zipimporter.java @@ -25,6 +25,7 @@ import org.python.core.PyType; import org.python.core.PyUnicode; import org.python.core.Traverseproc; import org.python.core.Visitproc; +import org.python.core.imp; import org.python.core.util.FileUtil; import org.python.core.util.StringUtil; import org.python.core.util.importer; @@ -90,7 +91,7 @@ public class zipimporter extends importe @ExposedMethod final void zipimporter___init__(PyObject[] args, String[] kwds) { ArgParser ap = new ArgParser("__init__", args, kwds, new String[] {"path"}); - String path = Py.fileSystemDecode(ap.getPyObject(0)); + String path = imp.fileSystemDecode(ap.getPyObject(0)); zipimporter___init__(path); }