TextGenWebUI API Compatible

[sfiannaca]

Just for fun, I played again with HTTP calls and crafted this raw and rough Dart file to play with TextGenWebUi ( openai api compatible exts)

import 'package:http/http.dart' as http;
import 'dart:async';
import 'dart:convert';

// interface:
// https://github.com/oobabooga/text-generation-webui/wiki/12-%E2%80%90-OpenAI-API#openai-compatible-api

void main() async {
  final prompt = "Why is the sky blue?:\n\n1.";
  final url = "http://192.168.1.100:5000/v1/chat/completions";
  final requestBody = 
    {
    "messages": [
      {
        "role": "user",
        "content": prompt
      }
    ],
    "mode": "chat",
    "character": "Example",
    "stream": true
  };
  gen() async* {
    final request = http.Request('POST', Uri.parse(url));
    request.headers['Content-Type'] = 'application/json';
    request.body = jsonEncode(requestBody);
    final response = await http.Client().send(request);

    print('Response status: ${response.statusCode}');
    if (response.statusCode == 200) {
      final controller = StreamController<Map<String, dynamic>>();
      response.stream.transform(utf8.decoder).listen((data) {
        if (data.isNotEmpty) {
          var jsonData = data;
          if (data.startsWith('data: ')) {
            jsonData = data.substring(6);
          }

          controller.add(jsonDecode(jsonData));
        }
      });
      yield* controller.stream;
      
    } else {
      throw Exception('Error on response');
    }
  }
  final streamController = StreamController<Map<String, dynamic>>();
  final apiResponseStream = gen();

  apiResponseStream.listen(
    (data) {
      print('Received data: $data');
    },
    onError: (error) {
      print("Error: $error");
    },
    onDone: () {
      print("API request completed.");
      streamController.close();
    },
  );

  await streamController.stream.drain();
  print("All data processed.");
}

this is the output:

Connecting to VM Service at ws://127.0.0.1:44193/KfsHd7TGUow=/ws
Response status: 200
Received data: {id: chatcmpl-1700497327248584448, object: chat.completions.chunk, created: 1700497327, model: TheBloke_dolphin-2.2.1-mistral-7B-GPTQ_gptq-4bit-32g-actorder_True, choices: [{index: 0, finish_reason: null, message: {role: assistant, content: }, delta: {role: assistant, content: }}]}
Received data: {id: chatcmpl-1700497327248584448, object: chat.completions.chunk, created: 1700497327, model: TheBloke_dolphin-2.2.1-mistral-7B-GPTQ_gptq-4bit-32g-actorder_True, choices: [{index: 0, finish_reason: null, message: {role: assistant, content: The}, delta: {role: assistant, content: The}}]}
Received data: {id: chatcmpl-1700497327248584448, object: chat.completions.chunk, created: 1700497327, model: TheBloke_dolphin-2.2.1-mistral-7B-GPTQ_gptq-4bit-32g-actorder_True, choices: [{index: 0, finish_reason: null, message: {role: assistant, content:  sky is blue}, delta: {role: assistant, content:  sky is blue}}]}
Received data: {id: chatcmpl-1700497327248584448, object: chat.completions.chunk, created: 1700497327, model: TheBloke_dolphin-2.2.1-mistral-7B-GPTQ_gptq-4bit-32g-actorder_True, choices: [{index: 0, finish_reason: null, message: {role: assistant, content:  because of Ray}, delta: {role: assistant, content:  because of Ray}}]}
Received data: {id: chatcmpl-1700497327248584448, object: chat.completions.chunk, created: 1700497327, model: TheBloke_dolphin-2.2.1-mistral-7B-GPTQ_gptq-4bit-32g-actorder_True, choices: [{index: 0, finish_reason: null, message: {role: assistant, content: leigh scattering}, delta: {role: assistant, content: leigh scattering}}]}
Received data: {id: chatcmpl-1700497327248584448, object: chat.completions.chunk, created: 1700497327, model: TheBloke_dolphin-2.2.1-mistral-7B-GPTQ_gptq-4bit-32g-actorder_True, choices: [{index: 0, finish_reason: null, message: {role: assistant, content: .
2}, delta: {role: assistant, content: .
2}}]}
Received data: {id: chatcmpl-1700497327248584448, object: chat.completions.chunk, created: 1700497327, model: TheBloke_dolphin-2.2.1-mistral-7B-GPTQ_gptq-4bit-32g-actorder_True, choices: [{index: 0, finish_reason: null, message: {role: assistant, content: .}, delta: {role: assistant, content: .}}]}
Received data: {id: chatcmpl-1700497327248584448, object: chat.completions.chunk, created: 1700497327, model: TheBloke_dolphin-2.2.1-mistral-7B-GPTQ_gptq-4bit-32g-actorder_True, choices: [{index: 0, finish_reason: stop, message: {role: assistant, content: }, delta: {role: assistant, content: }}], usage: {prompt_tokens: 237, completion_tokens: 15, total_tokens: 252}}

Exited.

[danemadsen]

thats cool. I will be eventually adding that API in because the Ollama team have expressed desire to make theirs have parity with it