Blame external/pybind11/tests/test_embed/test_interpreter.cpp

Packit 534379
#include <pybind11/embed.h>
Packit 534379
Packit 534379
#ifdef _MSC_VER
Packit 534379
// Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to catch
Packit 534379
// 2.0.1; this should be fixed in the next catch release after 2.0.1).
Packit 534379
#  pragma warning(disable: 4996)
Packit 534379
#endif
Packit 534379
Packit 534379
#include <catch.hpp>
Packit 534379
Packit 534379
#include <thread>
Packit 534379
#include <fstream>
Packit 534379
#include <functional>
Packit 534379
Packit 534379
namespace py = pybind11;
Packit 534379
using namespace py::literals;
Packit 534379
Packit 534379
class Widget {
Packit 534379
public:
Packit 534379
    Widget(std::string message) : message(message) { }
Packit 534379
    virtual ~Widget() = default;
Packit 534379
Packit 534379
    std::string the_message() const { return message; }
Packit 534379
    virtual int the_answer() const = 0;
Packit 534379
Packit 534379
private:
Packit 534379
    std::string message;
Packit 534379
};
Packit 534379
Packit 534379
class PyWidget final : public Widget {
Packit 534379
    using Widget::Widget;
Packit 534379
Packit 534379
    int the_answer() const override { PYBIND11_OVERLOAD_PURE(int, Widget, the_answer); }
Packit 534379
};
Packit 534379
Packit 534379
PYBIND11_EMBEDDED_MODULE(widget_module, m) {
Packit 534379
    py::class_<Widget, PyWidget>(m, "Widget")
Packit 534379
        .def(py::init<std::string>())
Packit 534379
        .def_property_readonly("the_message", &Widget::the_message);
Packit 534379
Packit 534379
    m.def("add", [](int i, int j) { return i + j; });
Packit 534379
}
Packit 534379
Packit 534379
PYBIND11_EMBEDDED_MODULE(throw_exception, ) {
Packit 534379
    throw std::runtime_error("C++ Error");
Packit 534379
}
Packit 534379
Packit 534379
PYBIND11_EMBEDDED_MODULE(throw_error_already_set, ) {
Packit 534379
    auto d = py::dict();
Packit 534379
    d["missing"].cast<py::object>();
Packit 534379
}
Packit 534379
Packit 534379
TEST_CASE("Pass classes and data between modules defined in C++ and Python") {
Packit 534379
    auto module = py::module::import("test_interpreter");
Packit 534379
    REQUIRE(py::hasattr(module, "DerivedWidget"));
Packit 534379
Packit 534379
    auto locals = py::dict("hello"_a="Hello, World!", "x"_a=5, **module.attr("__dict__"));
Packit 534379
    py::exec(R"(
Packit 534379
        widget = DerivedWidget("{} - {}".format(hello, x))
Packit 534379
        message = widget.the_message
Packit 534379
    )", py::globals(), locals);
Packit 534379
    REQUIRE(locals["message"].cast<std::string>() == "Hello, World! - 5");
Packit 534379
Packit 534379
    auto py_widget = module.attr("DerivedWidget")("The question");
Packit 534379
    auto message = py_widget.attr("the_message");
Packit 534379
    REQUIRE(message.cast<std::string>() == "The question");
Packit 534379
Packit 534379
    const auto &cpp_widget = py_widget.cast<const Widget &>();
Packit 534379
    REQUIRE(cpp_widget.the_answer() == 42);
Packit 534379
}
Packit 534379
Packit 534379
TEST_CASE("Import error handling") {
Packit 534379
    REQUIRE_NOTHROW(py::module::import("widget_module"));
Packit 534379
    REQUIRE_THROWS_WITH(py::module::import("throw_exception"),
Packit 534379
                        "ImportError: C++ Error");
Packit 534379
    REQUIRE_THROWS_WITH(py::module::import("throw_error_already_set"),
Packit 534379
                        Catch::Contains("ImportError: KeyError"));
Packit 534379
}
Packit 534379
Packit 534379
TEST_CASE("There can be only one interpreter") {
Packit 534379
    static_assert(std::is_move_constructible<py::scoped_interpreter>::value, "");
Packit 534379
    static_assert(!std::is_move_assignable<py::scoped_interpreter>::value, "");
Packit 534379
    static_assert(!std::is_copy_constructible<py::scoped_interpreter>::value, "");
Packit 534379
    static_assert(!std::is_copy_assignable<py::scoped_interpreter>::value, "");
Packit 534379
Packit 534379
    REQUIRE_THROWS_WITH(py::initialize_interpreter(), "The interpreter is already running");
Packit 534379
    REQUIRE_THROWS_WITH(py::scoped_interpreter(), "The interpreter is already running");
Packit 534379
Packit 534379
    py::finalize_interpreter();
Packit 534379
    REQUIRE_NOTHROW(py::scoped_interpreter());
Packit 534379
    {
Packit 534379
        auto pyi1 = py::scoped_interpreter();
Packit 534379
        auto pyi2 = std::move(pyi1);
Packit 534379
    }
Packit 534379
    py::initialize_interpreter();
Packit 534379
}
Packit 534379
Packit 534379
bool has_pybind11_internals_builtin() {
Packit 534379
    auto builtins = py::handle(PyEval_GetBuiltins());
Packit 534379
    return builtins.contains(PYBIND11_INTERNALS_ID);
Packit 534379
};
Packit 534379
Packit 534379
bool has_pybind11_internals_static() {
Packit 534379
    auto **&ipp = py::detail::get_internals_pp();
Packit 534379
    return ipp && *ipp;
Packit 534379
}
Packit 534379
Packit 534379
TEST_CASE("Restart the interpreter") {
Packit 534379
    // Verify pre-restart state.
Packit 534379
    REQUIRE(py::module::import("widget_module").attr("add")(1, 2).cast<int>() == 3);
Packit 534379
    REQUIRE(has_pybind11_internals_builtin());
Packit 534379
    REQUIRE(has_pybind11_internals_static());
Packit 534379
    REQUIRE(py::module::import("external_module").attr("A")(123).attr("value").cast<int>() == 123);
Packit 534379
Packit 534379
    // local and foreign module internals should point to the same internals:
Packit 534379
    REQUIRE(reinterpret_cast<uintptr_t>(*py::detail::get_internals_pp()) ==
Packit 534379
            py::module::import("external_module").attr("internals_at")().cast<uintptr_t>());
Packit 534379
Packit 534379
    // Restart the interpreter.
Packit 534379
    py::finalize_interpreter();
Packit 534379
    REQUIRE(Py_IsInitialized() == 0);
Packit 534379
Packit 534379
    py::initialize_interpreter();
Packit 534379
    REQUIRE(Py_IsInitialized() == 1);
Packit 534379
Packit 534379
    // Internals are deleted after a restart.
Packit 534379
    REQUIRE_FALSE(has_pybind11_internals_builtin());
Packit 534379
    REQUIRE_FALSE(has_pybind11_internals_static());
Packit 534379
    pybind11::detail::get_internals();
Packit 534379
    REQUIRE(has_pybind11_internals_builtin());
Packit 534379
    REQUIRE(has_pybind11_internals_static());
Packit 534379
    REQUIRE(reinterpret_cast<uintptr_t>(*py::detail::get_internals_pp()) ==
Packit 534379
            py::module::import("external_module").attr("internals_at")().cast<uintptr_t>());
Packit 534379
Packit 534379
    // Make sure that an interpreter with no get_internals() created until finalize still gets the
Packit 534379
    // internals destroyed
Packit 534379
    py::finalize_interpreter();
Packit 534379
    py::initialize_interpreter();
Packit 534379
    bool ran = false;
Packit 534379
    py::module::import("__main__").attr("internals_destroy_test") =
Packit 534379
        py::capsule(&ran, [](void *ran) { py::detail::get_internals(); *static_cast<bool *>(ran) = true; });
Packit 534379
    REQUIRE_FALSE(has_pybind11_internals_builtin());
Packit 534379
    REQUIRE_FALSE(has_pybind11_internals_static());
Packit 534379
    REQUIRE_FALSE(ran);
Packit 534379
    py::finalize_interpreter();
Packit 534379
    REQUIRE(ran);
Packit 534379
    py::initialize_interpreter();
Packit 534379
    REQUIRE_FALSE(has_pybind11_internals_builtin());
Packit 534379
    REQUIRE_FALSE(has_pybind11_internals_static());
Packit 534379
Packit 534379
    // C++ modules can be reloaded.
Packit 534379
    auto cpp_module = py::module::import("widget_module");
Packit 534379
    REQUIRE(cpp_module.attr("add")(1, 2).cast<int>() == 3);
Packit 534379
Packit 534379
    // C++ type information is reloaded and can be used in python modules.
Packit 534379
    auto py_module = py::module::import("test_interpreter");
Packit 534379
    auto py_widget = py_module.attr("DerivedWidget")("Hello after restart");
Packit 534379
    REQUIRE(py_widget.attr("the_message").cast<std::string>() == "Hello after restart");
Packit 534379
}
Packit 534379
Packit 534379
TEST_CASE("Subinterpreter") {
Packit 534379
    // Add tags to the modules in the main interpreter and test the basics.
Packit 534379
    py::module::import("__main__").attr("main_tag") = "main interpreter";
Packit 534379
    {
Packit 534379
        auto m = py::module::import("widget_module");
Packit 534379
        m.attr("extension_module_tag") = "added to module in main interpreter";
Packit 534379
Packit 534379
        REQUIRE(m.attr("add")(1, 2).cast<int>() == 3);
Packit 534379
    }
Packit 534379
    REQUIRE(has_pybind11_internals_builtin());
Packit 534379
    REQUIRE(has_pybind11_internals_static());
Packit 534379
Packit 534379
    /// Create and switch to a subinterpreter.
Packit 534379
    auto main_tstate = PyThreadState_Get();
Packit 534379
    auto sub_tstate = Py_NewInterpreter();
Packit 534379
Packit 534379
    // Subinterpreters get their own copy of builtins. detail::get_internals() still
Packit 534379
    // works by returning from the static variable, i.e. all interpreters share a single
Packit 534379
    // global pybind11::internals;
Packit 534379
    REQUIRE_FALSE(has_pybind11_internals_builtin());
Packit 534379
    REQUIRE(has_pybind11_internals_static());
Packit 534379
Packit 534379
    // Modules tags should be gone.
Packit 534379
    REQUIRE_FALSE(py::hasattr(py::module::import("__main__"), "tag"));
Packit 534379
    {
Packit 534379
        auto m = py::module::import("widget_module");
Packit 534379
        REQUIRE_FALSE(py::hasattr(m, "extension_module_tag"));
Packit 534379
Packit 534379
        // Function bindings should still work.
Packit 534379
        REQUIRE(m.attr("add")(1, 2).cast<int>() == 3);
Packit 534379
    }
Packit 534379
Packit 534379
    // Restore main interpreter.
Packit 534379
    Py_EndInterpreter(sub_tstate);
Packit 534379
    PyThreadState_Swap(main_tstate);
Packit 534379
Packit 534379
    REQUIRE(py::hasattr(py::module::import("__main__"), "main_tag"));
Packit 534379
    REQUIRE(py::hasattr(py::module::import("widget_module"), "extension_module_tag"));
Packit 534379
}
Packit 534379
Packit 534379
TEST_CASE("Execution frame") {
Packit 534379
    // When the interpreter is embedded, there is no execution frame, but `py::exec`
Packit 534379
    // should still function by using reasonable globals: `__main__.__dict__`.
Packit 534379
    py::exec("var = dict(number=42)");
Packit 534379
    REQUIRE(py::globals()["var"]["number"].cast<int>() == 42);
Packit 534379
}
Packit 534379
Packit 534379
TEST_CASE("Threads") {
Packit 534379
    // Restart interpreter to ensure threads are not initialized
Packit 534379
    py::finalize_interpreter();
Packit 534379
    py::initialize_interpreter();
Packit 534379
    REQUIRE_FALSE(has_pybind11_internals_static());
Packit 534379
Packit 534379
    constexpr auto num_threads = 10;
Packit 534379
    auto locals = py::dict("count"_a=0);
Packit 534379
Packit 534379
    {
Packit 534379
        py::gil_scoped_release gil_release{};
Packit 534379
        REQUIRE(has_pybind11_internals_static());
Packit 534379
Packit 534379
        auto threads = std::vector<std::thread>();
Packit 534379
        for (auto i = 0; i < num_threads; ++i) {
Packit 534379
            threads.emplace_back([&]() {
Packit 534379
                py::gil_scoped_acquire gil{};
Packit 534379
                locals["count"] = locals["count"].cast<int>() + 1;
Packit 534379
            });
Packit 534379
        }
Packit 534379
Packit 534379
        for (auto &thread : threads) {
Packit 534379
            thread.join();
Packit 534379
        }
Packit 534379
    }
Packit 534379
Packit 534379
    REQUIRE(locals["count"].cast<int>() == num_threads);
Packit 534379
}
Packit 534379
Packit 534379
// Scope exit utility https://stackoverflow.com/a/36644501/7255855
Packit 534379
struct scope_exit {
Packit 534379
    std::function<void()> f_;
Packit 534379
    explicit scope_exit(std::function<void()> f) noexcept : f_(std::move(f)) {}
Packit 534379
    ~scope_exit() { if (f_) f_(); }
Packit 534379
};
Packit 534379
Packit 534379
TEST_CASE("Reload module from file") {
Packit 534379
    // Disable generation of cached bytecode (.pyc files) for this test, otherwise
Packit 534379
    // Python might pick up an old version from the cache instead of the new versions
Packit 534379
    // of the .py files generated below
Packit 534379
    auto sys = py::module::import("sys");
Packit 534379
    bool dont_write_bytecode = sys.attr("dont_write_bytecode").cast<bool>();
Packit 534379
    sys.attr("dont_write_bytecode") = true;
Packit 534379
    // Reset the value at scope exit
Packit 534379
    scope_exit reset_dont_write_bytecode([&]() {
Packit 534379
        sys.attr("dont_write_bytecode") = dont_write_bytecode;
Packit 534379
    });
Packit 534379
Packit 534379
    std::string module_name = "test_module_reload";
Packit 534379
    std::string module_file = module_name + ".py";
Packit 534379
Packit 534379
    // Create the module .py file
Packit 534379
    std::ofstream test_module(module_file);
Packit 534379
    test_module << "def test():\n";
Packit 534379
    test_module << "    return 1\n";
Packit 534379
    test_module.close();
Packit 534379
    // Delete the file at scope exit
Packit 534379
    scope_exit delete_module_file([&]() {
Packit 534379
        std::remove(module_file.c_str());
Packit 534379
    });
Packit 534379
Packit 534379
    // Import the module from file
Packit 534379
    auto module = py::module::import(module_name.c_str());
Packit 534379
    int result = module.attr("test")().cast<int>();
Packit 534379
    REQUIRE(result == 1);
Packit 534379
Packit 534379
    // Update the module .py file with a small change
Packit 534379
    test_module.open(module_file);
Packit 534379
    test_module << "def test():\n";
Packit 534379
    test_module << "    return 2\n";
Packit 534379
    test_module.close();
Packit 534379
Packit 534379
    // Reload the module
Packit 534379
    module.reload();
Packit 534379
    result = module.attr("test")().cast<int>();
Packit 534379
    REQUIRE(result == 2);
Packit 534379
}