Class _LocalModel

constructor

• new _LocalModel(args: LocalModelArgs): _LocalModel

  Parameters

  • args: LocalModelArgs

  Returns _LocalModel

backend

ReadonlycomponentType

id

quantization

defaultSystemMessage

• defaultSystemMessage(): undefined | string

  Returns undefined | string

dispose

• dispose(): Promise<void>

  Returns Promise<void>

infer

• infer(
      args: { messages: Message[]; reasoning?: boolean; tools: Tool[] },
  ): Promise<AsyncIterable<MessageOutput, any, any>>

  Parameters

  • args: { messages: Message[]; reasoning?: boolean; tools: Tool[] }

  Returns Promise<AsyncIterable<MessageOutput, any, any>>

init

• init(args: { genConfig?: GenerationConfig; runtime: Runtime }): Promise<void>

  Parameters

  • args: { genConfig?: GenerationConfig; runtime: Runtime }

  Returns Promise<void>

isBor

• isBor(tok: string): boolean

  Parameters

  • tok: string

  Returns boolean

isBos

• isBos(tok: string): boolean

  Parameters

  • tok: string

  Returns boolean

isBotc

• isBotc(tok: string): boolean

  Parameters

  • tok: string

  Returns boolean

isEor

• isEor(tok: string): boolean

  Parameters

  • tok: string

  Returns boolean

isEos

• isEos(tok: string): boolean

  Parameters

  • tok: string

  Returns boolean

isEotc

• isEotc(tok: string): boolean

  Parameters

  • tok: string

  Returns boolean