rust/hg-cpython/src/copy_tracing.rs
author Rapha?l Gom?s <rgomes@octobus.net>
Wed, 28 Jun 2023 16:43:39 +0200
changeset 51192 f6403bcd9f96
parent 50976 4c5f6e95df84
permissions -rw-r--r--
rust-index: synchronize remove to Rust index Future steps will bring the two indexes further together until we can rip the C index entirely when running Rust code.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     1
use cpython::ObjectProtocol;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     2
use cpython::PyBytes;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     3
use cpython::PyDict;
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
     4
use cpython::PyDrop;
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     5
use cpython::PyList;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     6
use cpython::PyModule;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     7
use cpython::PyObject;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     8
use cpython::PyResult;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     9
use cpython::PyTuple;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    10
use cpython::Python;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    11
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    12
use hg::copy_tracing::ChangedFiles;
46587
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
    13
use hg::copy_tracing::CombineChangesetCopies;
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    14
use hg::Revision;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    15
47953
8f031a274cd6 rust: Move PyBytesWithData out of copy-tracing code
Simon Sapin <simon.sapin@octobus.net>
parents: 46590
diff changeset
    16
use crate::pybytes_deref::PyBytesDeref;
50976
4c5f6e95df84 rust: make `Revision` a newtype
Rapha?l Gom?s <rgomes@octobus.net>
parents: 49933
diff changeset
    17
use crate::PyRevision;
46589
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    18
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    19
/// Combines copies information contained into revision `revs` to build a copy
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    20
/// map.
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    21
///
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    22
/// See mercurial/copies.py for details
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    23
pub fn combine_changeset_copies_wrapper(
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    24
    py: Python,
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    25
    revs: PyList,
46149
294d5aca4ff5 copies: iterate over children directly (instead of parents)
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 46057
diff changeset
    26
    children_count: PyDict,
50976
4c5f6e95df84 rust: make `Revision` a newtype
Rapha?l Gom?s <rgomes@octobus.net>
parents: 49933
diff changeset
    27
    target_rev: PyRevision,
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    28
    rev_info: PyObject,
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    29
    multi_thread: bool,
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    30
) -> PyResult<PyDict> {
50976
4c5f6e95df84 rust: make `Revision` a newtype
Rapha?l Gom?s <rgomes@octobus.net>
parents: 49933
diff changeset
    31
    let target_rev = Revision(target_rev.0);
46587
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
    32
    let children_count = children_count
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    33
        .items(py)
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    34
        .iter()
50976
4c5f6e95df84 rust: make `Revision` a newtype
Rapha?l Gom?s <rgomes@octobus.net>
parents: 49933
diff changeset
    35
        .map(|(k, v)| {
4c5f6e95df84 rust: make `Revision` a newtype
Rapha?l Gom?s <rgomes@octobus.net>
parents: 49933
diff changeset
    36
            Ok((Revision(k.extract::<PyRevision>(py)?.0), v.extract(py)?))
4c5f6e95df84 rust: make `Revision` a newtype
Rapha?l Gom?s <rgomes@octobus.net>
parents: 49933
diff changeset
    37
        })
46587
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
    38
        .collect::<PyResult<_>>()?;
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
    39
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
    40
    /// (Revision number, parent 1, parent 2, copy data for this revision)
46589
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    41
    type RevInfo<Bytes> = (Revision, Revision, Revision, Option<Bytes>);
46587
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
    42
46589
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    43
    let revs_info =
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    44
        revs.iter(py).map(|rev_py| -> PyResult<RevInfo<PyBytes>> {
50976
4c5f6e95df84 rust: make `Revision` a newtype
Rapha?l Gom?s <rgomes@octobus.net>
parents: 49933
diff changeset
    45
            let rev = Revision(rev_py.extract::<PyRevision>(py)?.0);
46589
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    46
            let tuple: PyTuple =
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    47
                rev_info.call(py, (rev_py,), None)?.cast_into(py)?;
50976
4c5f6e95df84 rust: make `Revision` a newtype
Rapha?l Gom?s <rgomes@octobus.net>
parents: 49933
diff changeset
    48
            let p1 =
4c5f6e95df84 rust: make `Revision` a newtype
Rapha?l Gom?s <rgomes@octobus.net>
parents: 49933
diff changeset
    49
                Revision(tuple.get_item(py, 0).extract::<PyRevision>(py)?.0);
4c5f6e95df84 rust: make `Revision` a newtype
Rapha?l Gom?s <rgomes@octobus.net>
parents: 49933
diff changeset
    50
            let p2 =
4c5f6e95df84 rust: make `Revision` a newtype
Rapha?l Gom?s <rgomes@octobus.net>
parents: 49933
diff changeset
    51
                Revision(tuple.get_item(py, 1).extract::<PyRevision>(py)?.0);
46589
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    52
            let opt_bytes = tuple.get_item(py, 2).extract(py)?;
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    53
            Ok((rev, p1, p2, opt_bytes))
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
    54
        });
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    55
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    56
    let path_copies;
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    57
    if !multi_thread {
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    58
        let mut combine_changeset_copies =
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    59
            CombineChangesetCopies::new(children_count);
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    60
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    61
        for rev_info in revs_info {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    62
            let (rev, p1, p2, opt_bytes) = rev_info?;
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    63
            let files = match &opt_bytes {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    64
                Some(bytes) => ChangedFiles::new(bytes.data(py)),
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    65
                // Python None was extracted to Option::None,
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    66
                // meaning there was no copy data.
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    67
                None => ChangedFiles::new_empty(),
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    68
            };
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    69
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    70
            combine_changeset_copies.add_revision(rev, p1, p2, files)
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    71
        }
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    72
        path_copies = combine_changeset_copies.finish(target_rev)
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    73
    } else {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    74
        // Use a bounded channel to provide back-pressure:
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    75
        // if the child thread is slower to process revisions than this thread
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    76
        // is to gather data for them, an unbounded channel would keep
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    77
        // growing and eat memory.
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    78
        //
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    79
        // TODO: tweak the bound?
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    80
        let (rev_info_sender, rev_info_receiver) =
47953
8f031a274cd6 rust: Move PyBytesWithData out of copy-tracing code
Simon Sapin <simon.sapin@octobus.net>
parents: 46590
diff changeset
    81
            crossbeam_channel::bounded::<RevInfo<PyBytesDeref>>(1000);
46587
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
    82
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    83
        // This channel (going the other way around) however is unbounded.
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    84
        // If they were both bounded, there might potentially be deadlocks
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    85
        // where both channels are full and both threads are waiting on each
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    86
        // other.
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    87
        let (pybytes_sender, pybytes_receiver) =
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    88
            crossbeam_channel::unbounded();
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
    89
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    90
        // Start a thread that does CPU-heavy processing in parallel with the
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    91
        // loop below.
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    92
        //
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    93
        // If the parent thread panics, `rev_info_sender` will be dropped and
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    94
        // “disconnected”. `rev_info_receiver` will be notified of this and
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    95
        // exit its own loop.
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    96
        let thread = std::thread::spawn(move || {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    97
            let mut combine_changeset_copies =
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    98
                CombineChangesetCopies::new(children_count);
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
    99
            for (rev, p1, p2, opt_bytes) in rev_info_receiver {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   100
                let files = match &opt_bytes {
47953
8f031a274cd6 rust: Move PyBytesWithData out of copy-tracing code
Simon Sapin <simon.sapin@octobus.net>
parents: 46590
diff changeset
   101
                    Some(raw) => ChangedFiles::new(raw.as_ref()),
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   102
                    // Python None was extracted to Option::None,
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   103
                    // meaning there was no copy data.
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   104
                    None => ChangedFiles::new_empty(),
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   105
                };
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   106
                combine_changeset_copies.add_revision(rev, p1, p2, files);
46589
620c88fb42a2 copies-rust: introduce PyBytesWithData to reduce GIL requirement
Simon Sapin <simon-commits@exyr.org>
parents: 46588
diff changeset
   107
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   108
                // Send `PyBytes` back to the parent thread so the parent
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   109
                // thread can drop it. Otherwise the GIL would be implicitly
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   110
                // acquired here through `impl Drop for PyBytes`.
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   111
                if let Some(bytes) = opt_bytes {
49933
be3b545c5cff rust-clippy: fix remaining warnings in `hg-cpython`
Rapha?l Gom?s <rgomes@octobus.net>
parents: 47953
diff changeset
   112
                    if pybytes_sender.send(bytes.unwrap()).is_err() {
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   113
                        // The channel is disconnected, meaning the parent
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   114
                        // thread panicked or returned
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   115
                        // early through
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   116
                        // `?` to propagate a Python exception.
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   117
                        break;
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   118
                    }
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   119
                }
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   120
            }
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   121
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   122
            combine_changeset_copies.finish(target_rev)
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   123
        });
46587
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
   124
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   125
        for rev_info in revs_info {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   126
            let (rev, p1, p2, opt_bytes) = rev_info?;
47953
8f031a274cd6 rust: Move PyBytesWithData out of copy-tracing code
Simon Sapin <simon.sapin@octobus.net>
parents: 46590
diff changeset
   127
            let opt_bytes = opt_bytes.map(|b| PyBytesDeref::new(py, b));
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   128
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   129
            // We’d prefer to avoid the child thread calling into Python code,
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   130
            // but this avoids a potential deadlock on the GIL if it does:
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   131
            py.allow_threads(|| {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   132
                rev_info_sender.send((rev, p1, p2, opt_bytes)).expect(
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   133
                    "combine_changeset_copies: channel is disconnected",
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   134
                );
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   135
            });
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   136
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   137
            // Drop anything in the channel, without blocking
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   138
            for pybytes in pybytes_receiver.try_iter() {
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   139
                pybytes.release_ref(py)
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   140
            }
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   141
        }
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   142
        // We’d prefer to avoid the child thread calling into Python code,
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   143
        // but this avoids a potential deadlock on the GIL if it does:
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   144
        path_copies = py.allow_threads(|| {
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   145
            // Disconnect the channel to signal the child thread to stop:
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   146
            // the `for … in rev_info_receiver` loop will end.
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   147
            drop(rev_info_sender);
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   148
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   149
            // Wait for the child thread to stop, and propagate any panic.
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   150
            thread.join().unwrap_or_else(|panic_payload| {
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   151
                std::panic::resume_unwind(panic_payload)
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   152
            })
46590
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   153
        });
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   154
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   155
        // Drop anything left in the channel
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   156
        for pybytes in pybytes_receiver.iter() {
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   157
            pybytes.release_ref(py)
8d20abed6a1e copies-rust: send PyBytes values back be dropped ino the parent thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46589
diff changeset
   158
        }
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   159
    };
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   160
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   161
    let out = PyDict::new(py);
46587
cb4b0b0c6de4 copies-rust: split up combine_changeset_copies function into a struct
Simon Sapin <simon.sapin@octobus.net>
parents: 46569
diff changeset
   162
    for (dest, source) in path_copies.into_iter() {
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   163
        out.set_item(
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   164
            py,
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   165
            PyBytes::new(py, &dest.into_vec()),
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   166
            PyBytes::new(py, &source.into_vec()),
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   167
        )?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   168
    }
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   169
    Ok(out)
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   170
}
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   171
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   172
/// Create the module, with `__package__` given from parent
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   173
pub fn init_module(py: Python, package: &str) -> PyResult<PyModule> {
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   174
    let dotted_name = &format!("{}.copy_tracing", package);
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   175
    let m = PyModule::new(py, dotted_name)?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   176
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   177
    m.add(py, "__package__", package)?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   178
    m.add(py, "__doc__", "Copy tracing - Rust implementation")?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   179
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   180
    m.add(
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   181
        py,
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   182
        "combine_changeset_copies",
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   183
        py_fn!(
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   184
            py,
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   185
            combine_changeset_copies_wrapper(
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   186
                revs: PyList,
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   187
                children: PyDict,
50976
4c5f6e95df84 rust: make `Revision` a newtype
Rapha?l Gom?s <rgomes@octobus.net>
parents: 49933
diff changeset
   188
                target_rev: PyRevision,
46588
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   189
                rev_info: PyObject,
47557ea79fc7 copies-rust: move CPU-heavy Rust processing into a child thread
Simon Sapin <simon.sapin@octobus.net>
parents: 46587
diff changeset
   190
                multi_thread: bool
45945
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   191
            )
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   192
        ),
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   193
    )?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   194
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   195
    let sys = PyModule::import(py, "sys")?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   196
    let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   197
    sys_modules.set_item(py, dotted_name, &m)?;
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   198
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   199
    Ok(m)
50c5ee3bdf9a copies: introduce the hg-cpython wrapper for `combine_changeset_copies`
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
   200
}