-
Notifications
You must be signed in to change notification settings - Fork 20
/
fmbench_predictor.py
61 lines (50 loc) · 1.73 KB
/
fmbench_predictor.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
import pandas as pd
from datetime import datetime
from typing import Dict, Optional
from abc import ABC, abstractmethod, abstractproperty
class FMBenchPredictor(ABC):
@abstractmethod
def __init__(self,
endpoint_name: str,
inference_spec: Optional[Dict],
metadata: Optional[Dict]):
pass
@abstractmethod
def get_prediction(self, payload: Dict) -> Dict:
pass
@abstractmethod
def calculate_cost(self,
instance_type: str,
instance_count: int,
config: Dict,
duration: float,
metrics: Dict) -> float:
"""Represents the function to calculate the
cost of each experiment run.
"""
pass
@abstractmethod
def get_metrics(self,
start_time: datetime,
end_time: datetime,
period: int = 60) -> pd.DataFrame:
"""Represents the function to calculate the
metrics for each endpoint
"""
pass
@abstractproperty
def endpoint_name(self) -> str:
"""The endpoint name property."""
pass
@abstractproperty
def inference_parameters(self) -> Dict:
"""The inference parameters property."""
pass
class FMBenchPredictionResponse(dict):
def __init__(self, *k, **kwargs):
self.__dict__ = self
self.__dict__['response_json'] = kwargs['response_json']
self.__dict__['latency'] = kwargs['latency']
self.__dict__['prompt_tokens'] = kwargs['prompt_tokens']
self.__dict__['completion_tokens'] = kwargs['completion_tokens']
super().__init__(*k, **kwargs)