5441130e0a
Add a stream parser to extract citations (and others) from a stream. This support cases where markers are split in differen tokens. Codex never manage to make this code work so everything was done manually. Please review correctly and do not touch this part of the code without a very clear understanding of it
23 lines
793 B
Rust
23 lines
793 B
Rust
mod assistant_text;
|
|
mod citation;
|
|
mod inline_hidden_tag;
|
|
mod proposed_plan;
|
|
mod stream_text;
|
|
mod tagged_line_parser;
|
|
mod utf8_stream;
|
|
|
|
pub use assistant_text::AssistantTextChunk;
|
|
pub use assistant_text::AssistantTextStreamParser;
|
|
pub use citation::CitationStreamParser;
|
|
pub use citation::strip_citations;
|
|
pub use inline_hidden_tag::ExtractedInlineTag;
|
|
pub use inline_hidden_tag::InlineHiddenTagParser;
|
|
pub use inline_hidden_tag::InlineTagSpec;
|
|
pub use proposed_plan::ProposedPlanParser;
|
|
pub use proposed_plan::ProposedPlanSegment;
|
|
pub use proposed_plan::extract_proposed_plan_text;
|
|
pub use proposed_plan::strip_proposed_plan_blocks;
|
|
pub use stream_text::StreamTextChunk;
|
|
pub use stream_text::StreamTextParser;
|
|
pub use utf8_stream::Utf8StreamParser;
|
|
pub use utf8_stream::Utf8StreamParserError;
|