Streaming

use futures::StreamExt;
use langchainx::{
    chain::{Chain, LLMChainBuilder},
    prompt_args,
};

let chain = LLMChainBuilder::new().llm(llm).prompt(prompt).build()?;

let mut stream = chain.stream(prompt_args! { "input" => "Write a poem" }).await?;

while let Some(chunk) = stream.next().await {
    match chunk {
        Ok(data) => print!("{}", data.content),
        Err(e) => eprintln!("error: {e}"),
    }
}
println!(); // final newline

use langchainx::language_models::llm::LLM;
use langchainx::schemas::Message;

let mut stream = llm.stream(&[Message::new_human_message("Hello")]).await?;
while let Some(chunk) = stream.next().await {
    let data = chunk?;
    if !data.content.is_empty() {
        print!("{}", data.content);
    }
}

// WRONG — streaming_func is a side-channel callback in CallOptions
// It will be removed in JOB-253. Do not use in new code.
let options = CallOptions::new()
    .with_streaming_func(|chunk| async move {
        print!("{chunk}");
        Ok(())
    });
let llm = OpenAI::default().with_options(options);
let result = llm.generate(&messages).await?; // side-effects in callback, result separate

Problems with streaming_func:

• Runtime state inside a config struct
• Arc<Mutex<FnMut>> contamination
• Result (full string) and display (callback) are completely separate
• Cannot be tested without an async closure mock

pub struct StreamData {
    pub value: Value,      // raw JSON from the provider
    pub tokens: Option<TokenUsage>,
    pub content: String,   // the text chunk — this is what you usually want
}