rust/hg-pyo3/src/util.rs
author Georges Racinet <georges.racinet@cloudcrane.io>
Wed, 25 Dec 2024 13:29:56 +0100
changeset 52780 42b219a1404a
parent 52407 c5128c541021
child 52822 4f41a8acf350
permissions -rw-r--r--
rust-pyo3-revlog: InnerRevlog definition and constructor We felt that the `mmap_keeparound` naming was after all inappropriate: - it does nothing to keep the data alive. Instead, it requires the caller to keep it alive. - using it for mmap'ed data is certainly the main use case, but more generally, it is about extracting any Python buffer for direct usage as a bytes slice. Everything else is rather straightforward. This object has several layers of inner mutability, like its progenitor in `hg-cpython`.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
52780
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
     1
use pyo3::buffer::{Element, PyBuffer};
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
     2
use pyo3::exceptions::PyValueError;
52407
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
     3
use pyo3::prelude::*;
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
     4
use pyo3::types::PyDict;
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
     5
/// Create the module, with `__package__` given from parent
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
     6
///
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
     7
/// According to PyO3 documentation, which links to
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
     8
/// <https://github.com/PyO3/pyo3/issues/1517>, the same convoluted
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
     9
/// write to sys.modules has to be made as with the `cpython` crate.
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    10
pub(crate) fn new_submodule<'py>(
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    11
    py: Python<'py>,
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    12
    package_name: &str,
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    13
    name: &str,
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    14
) -> PyResult<Bound<'py, PyModule>> {
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    15
    let dotted_name = &format!("{}.{}", package_name, name);
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    16
    let m = PyModule::new(py, name)?;
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    17
    m.add("__package__", package_name)?;
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    18
    m.add("__doc__", "DAG operations - Rust implementation")?;
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    19
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    20
    let sys = PyModule::import(py, "sys")?;
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    21
    // according to the doc, we could make a static PyString out of
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    22
    // "modules" with the `intern!` macro, but this is used only at
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    23
    // registration so it may not be worth the effort.
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    24
    let sys_modules: Bound<'_, PyDict> = sys.getattr("modules")?.extract()?;
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    25
    sys_modules.set_item(dotted_name, &m)?;
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    26
    // Example C code (see pyexpat.c and import.c) will "give away the
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    27
    // reference", but we won't because it will be consumed once the
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    28
    // Rust PyObject is dropped.
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    29
    Ok(m)
c5128c541021 rust-pyo3: facility for submodule registration, using it for dagop
Georges Racinet <georges.racinet@cloudcrane.io>
parents:
diff changeset
    30
}
52780
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    31
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    32
/// Type shortcut for the kind of bytes slice trait objects that are used in
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    33
/// particular for mmap data
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    34
type BoxedBytesSlice =
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    35
    Box<dyn std::ops::Deref<Target = [u8]> + Send + Sync + 'static>;
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    36
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    37
/// Take a Python object backed by a Python buffer, and return the underlying
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    38
/// [`PyBuffer`] along with the Rust slice into said buffer.
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    39
///
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    40
/// The caller needs to make sure that the Python buffer is not freed before
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    41
/// the slice, otherwise we'd get a dangling pointer once the incoming
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    42
/// object is freed from Python side. This can be achieved by storing it a
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    43
/// Python object.
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    44
///
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    45
/// The typical use case is to extract mmap data to make it useable in the
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    46
/// constructs from the `hg` crate.
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    47
///
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    48
/// # Safety
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    49
///
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    50
/// The caller must make sure that the incoming Python object is kept around
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    51
/// for at least as long as the returned [`BoxedBytesSlice`].
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    52
// TODO in PyO3, we already get a reference with two lifetimes, and we
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    53
// could even take a `Borrowed<'a, 'py, T>`.
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    54
// So perhaps we could tie everything together with a lifetime so that is
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    55
// is, after all, safe, and this could be called something like `share_buffer`.
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    56
#[deny(unsafe_op_in_unsafe_fn)]
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    57
pub unsafe fn take_buffer_with_slice(
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    58
    data: &Bound<'_, PyAny>,
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    59
) -> PyResult<(PyBuffer<u8>, BoxedBytesSlice)> {
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    60
    let buf = PyBuffer::<u8>::get(data)?;
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    61
    let len = buf.item_count();
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    62
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    63
    // Build a slice from the buffer data
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    64
    let cbuf = buf.buf_ptr();
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    65
    let bytes = if std::mem::size_of::<u8>() == buf.item_size()
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    66
        && buf.is_c_contiguous()
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    67
        && u8::is_compatible_format(buf.format())
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    68
        && buf.dimensions() == 1
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    69
        && buf.readonly()
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    70
    {
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    71
        unsafe { std::slice::from_raw_parts(cbuf as *const u8, len) }
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    72
    } else {
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    73
        return Err(PyValueError::new_err(
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    74
            "buffer has an invalid memory representation",
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    75
        ));
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    76
    };
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    77
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    78
    Ok((buf, Box::new(bytes)))
42b219a1404a rust-pyo3-revlog: InnerRevlog definition and constructor
Georges Racinet <georges.racinet@cloudcrane.io>
parents: 52407
diff changeset
    79
}