| from functools import lru_cache |
|
|
| from .base import register_llm |
| from .llm_client import LLMFlaskClient |
|
|
|
|
| @lru_cache() |
| def _get_mistral_7b_instruct_server(host: str, port: int): |
| from .llm_server import LLMInstance, create_app |
| core = LLMInstance('Mistral-7B-Instruct-v0.1') |
| app = create_app(core) |
| app.run(host=host, port=port) |
|
|
|
|
| def ask_mistral_7b_instruct(message: str, **kwargs): |
| host, port = '0.0.0.0', 8001 |
| _get_mistral_7b_instruct_server(host, port) |
| client = LLMFlaskClient(host, port) |
| return client.run(message).strip() |
|
|
|
|
| register_llm('mistral-7b', ask_mistral_7b_instruct) |
|
|
