Streaming

import { createAnonymizerStream, AnonymizerStream, SentenceBuffer } from 'rehydra/streaming';

function createAnonymizerStream(config?: StreamConfig): Promise<AnonymizerStream>

import { createAnonymizerStream } from 'rehydra/streaming';

const stream = await createAnonymizerStream({
  anonymizer: { ner: { mode: 'quantized' } },
  onChunk: (e) => console.log(`Processed: ${e.totalEntities} entities`),
  onFinish: (e) => console.log(`Done: ${e.totalEntities} total`),
});

process.stdin.pipe(stream).pipe(process.stdout);

const stream = await createAnonymizerStream({ ... });

inputStream.pipe(stream).pipe(outputStream);

stream.on('finish', () => {
  const piiMap = stream.getPiiMap();
  const stats = stream.stats;
  console.log(`Entities: ${stats.totalEntities}, Time: ${stats.totalProcessingTimeMs}ms`);
});

new SentenceBuffer(anonymizer: Anonymizer, config: ResolvedBufferConfig)

interface StreamChunkEvent {
  anonymizedText: string;
  entities: DetectedEntity[];
  totalEntities: number;
  processingTimeMs: number;
}

interface StreamFinishEvent {
  totalEntities: number;
  piiMap?: EncryptedPIIMap;
  totalProcessingTimeMs: number;
}

interface FlushResult {
  anonymizedText: string;
  entities: DetectedEntity[];
}