Module Forester_search.TokenizerSource

Sourcemodule Set : sig ... end
Sourcetype loc =
  1. | In_frontmatter
  2. | In_mainmatter
Sourceval int_of_field_frontmatter : [< `attributions | `dates | `designated_parent | `metas | `number | `source_path | `tags | `taxon | `title | `uri ] -> int
Sourceval int_of_field_article : [< `backmatter | `frontmatter | `mainmatter ] -> int
Sourcetype token = {
  1. v : string;
  2. loc : loc;
}
Sourceval common_words : Set.t
Sourceval tokenize : string -> string list
Sourceval tokenize_content : int list -> loc -> Forester_core.Types.content -> (int list * string) list
Sourceval tokenize_vertex : int list -> loc -> Forester_core.Types.content Forester_core.Types.vertex -> (int list * string) list
Sourceval tokenize_attribution : int list -> loc -> Forester_core.Types.content Forester_core.Types.attribution -> (int list * string) list
Sourceval tokenize_frontmatter : int list -> Forester_core.Types.content Forester_core.Types.frontmatter -> (int list * string) list
Sourceval tokenize_article : Forester_core.Types.content Forester_core.Types.article -> (int list * string) list