Source file trace_replay_intf.ml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
(*
 * Copyright (c) 2018-2022 Tarides <contact@tarides.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 *)

module Config = struct
  type _ return_type =
    | Unit : unit return_type
    | Summary : Trace_stat_summary.t return_type

  type 'a config = {
    number_of_commits_to_replay : int;
    path_conversion : [ `None | `V1 | `V0_and_v1 | `V0 ];
    inode_config : int * int;
    store_type : [ `Pack | `Pack_layered | `Pack_mem ];
    replay_trace_path : string;
    artefacts_path : string;
    keep_store : bool;
    keep_stat_trace : bool;
    empty_blobs : bool;
    return_type : 'a return_type;
    gc_every : int;
    gc_distance_in_the_past : int;
    gc_wait_after : int;
    add_volume_every : int;
  }
  (** Replay configuration

      [replay_trace_path] points to a specific file that describes the sequence
      of operations to replay. You may download one of the following URLs. The
      smaller ones are prefix of the larger ones.

      - http://data.tarides.com/irmin/data4_10310commits.repr (0.3GB)
      - http://data.tarides.com/irmin/data4_100066commits.repr (2.9GB)
      - http://data.tarides.com/irmin/data_1343496commits.repr (102GB)

      [number_of_commits_to_replay] is the wished number of commits to replay.
      If the value is too high, the replay will stop when reaching the end of
      [replay_trace_path]. Pick a number of commits depending on the wished
      runtime. Here are some reference runtimes that were true for irmin 3.0:

      - [60_457] commits take 3 minutes
      - [500_000] commits take 1 hour
      - [1_343_496] commits take 5 hours

      [artefacts_path] is the destination for the stats trace and the store. If
      both [keep_store] and [keep_stat_trace] are false, the destination will be
      emptied at the end of the replay.

      [path_conversion] is the strategy for shortening the paths while
      replaying. Was useful when benchmarking irmin on flattened Tezos paths.

      [empty_blobs] make the replay to push the empty string as in all the
      blobs, instead of their actual value read in the trace.

      [inode_config] is a pair of ints that will be stored in the results of the
      replay.

      A GC is triggered every [gc_every] commits. When GC is triggered, we
      select a previous commit that is [gc_distance_in_the_past] commits away
      from the current head commit.

      The first GC will be started after [gc_distance_in_the_past + 1] commits
      were replayed. [gc_distance_in_the_past] only makes sense if [gc_every] is
      not [0].

      [gc_wait_after] defines how many commits separate the start of a GC and
      the moment we block to wait for it to finish. [0] means that we will only
      block when the next gc starts or at the end of the replay. This parameter
      only makes sense if [gc_every] is not [0]. *)
end

module type Config = module type of Config

include Config

module type Store = sig
  type store_config
  type key

  include
    Irmin.Generic_key.KV
      with type Schema.Contents.t = bytes
       and type commit_key = key
       and type node_key = key
       and type contents_key = key

  type on_commit := int -> Hash.t -> unit Lwt.t
  type on_end := unit -> unit Lwt.t

  val create_repo :
    root:string -> store_config -> (Repo.t * on_commit * on_end) Lwt.t

  val split : repo -> unit
  val add_volume : repo -> unit
  val gc_wait : repo -> unit Lwt.t

  type stats := Irmin_pack_unix.Stats.Latest_gc.stats

  val gc_run :
    ?finished:((stats, string) result -> unit Lwt.t) ->
    repo ->
    commit_key ->
    unit Lwt.t
end

module type Sigs = sig
  include
    Config
      with type 'a return_type = 'a return_type
       and type 'a config = 'a config

  module type Store = Store

  module Make (Store : Store) : sig
    include
      Config
        with type 'a return_type = 'a return_type
         and type 'a config = 'a config

    val run : Store.store_config -> 'a config -> 'a Lwt.t
  end
end