File size: 6,653 Bytes
74f2af5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 | #!/usr/bin/env python3
"""
Baseline Benchmark — Measure orchestrator latencies WITHOUT Phase 6/7
Test 30 queries (10 per complexity) to establish baseline latencies.
Then Phase 7 improvements can be compared against these numbers.
"""
import json
import time
import urllib.request
import urllib.error
# Test queries
QUERIES = {
"SIMPLE": [
"What is the speed of light?",
"Define entropy",
"Who is Albert Einstein?",
"What year was the Internet invented?",
"How high is Mount Everest?",
"What is the chemical formula for water?",
"Define photosynthesis",
"Who wrote Romeo and Juliet?",
"What is the capital of France?",
"How fast can a cheetah run?",
],
"MEDIUM": [
"How does quantum mechanics relate to consciousness?",
"What are the implications of artificial intelligence?",
"Compare classical and quantum computing",
"How do neural networks learn?",
"What is the relationship between energy and mass?",
"How does evolution explain biodiversity?",
"What are the main differences between mitochondria and chloroplasts?",
"How does feedback regulate biological systems?",
"What is the connection between sleep and memory consolidation?",
"How do economic systems balance growth and sustainability?",
],
"COMPLEX": [
"Can machines be truly conscious?",
"What is the nature of free will and how does it relate to determinism?",
"Is artificial intelligence the future of humanity?",
"How should AI be ethically governed?",
"What makes something morally right or wrong?",
"Can subjective experience be measured objectively?",
"How does quantum mechanics challenge our understanding of reality?",
"What is the relationship between language and thought?",
"How should society balance individual freedom with collective good?",
"Is human consciousness unique, or could machines achieve it?",
],
}
SERVER_URL = "http://localhost:7860"
def benchmark_queries():
"""Run baseline benchmark against all 30 queries."""
print("\n" + "="*70)
print("BASELINE BENCHMARK — Orchestrator WITHOUT Phase 6/7")
print("="*70)
results = {"SIMPLE": [], "MEDIUM": [], "COMPLEX": []}
# Check server (allow up to 180s for model loading on first startup)
print("\nChecking server status (waiting up to 180s for model load)...")
start_wait = time.time()
timeout_per_check = 10 # Each check waits 10s
max_total_wait = 180 # Total 3 minutes
response = None
while time.time() - start_wait < max_total_wait:
try:
response = urllib.request.urlopen(f"{SERVER_URL}/api/status", timeout=timeout_per_check)
status = json.loads(response.read().decode('utf-8'))
print(f" Server state: {status.get('state')}")
if status.get('state') != 'ready':
print(f" Waiting for server to reach 'ready' state...")
time.sleep(2)
continue
break # Server is ready!
except Exception as e:
elapsed = time.time() - start_wait
print(f" [{elapsed:.0f}s] Waiting for server... ({e})")
time.sleep(2)
continue
if response is None:
print(f" ERROR: Server never became available after {max_total_wait}s")
return results
# Run queries
total_start = time.time()
completed = 0
for complexity in ["SIMPLE", "MEDIUM", "COMPLEX"]:
print(f"\n[{complexity}] Testing {len(QUERIES[complexity])} queries:")
for i, query in enumerate(QUERIES[complexity], 1):
try:
start_time = time.time()
data = json.dumps({
"query": query,
"max_adapters": 2
}).encode('utf-8')
req = urllib.request.Request(
f"{SERVER_URL}/api/chat",
data=data,
headers={'Content-Type': 'application/json'}
)
response = urllib.request.urlopen(req, timeout=60)
result = json.loads(response.read().decode('utf-8'))
elapsed = time.time() - start_time
token_count = result.get('tokens', 0)
# Store result
results[complexity].append({
"query": query[:50],
"latency_ms": elapsed * 1000,
"tokens": token_count,
"success": True
})
print(f" [{i:2d}/10] {elapsed:6.1f}ms | {query[:40]}...")
completed += 1
except urllib.error.HTTPError as e:
print(f" [{i:2d}/10] HTTP {e.code} | {query[:40]}...")
results[complexity].append({
"query": query[:50],
"error": f"HTTP {e.code}",
"success": False
})
except Exception as e:
print(f" [{i:2d}/10] ERROR: {str(e)[:30]} | {query[:40]}...")
results[complexity].append({
"query": query[:50],
"error": str(e)[:50],
"success": False
})
# Summary
total_elapsed = time.time() - total_start
print(f"\n" + "="*70)
print(f"RESULTS: {completed}/30 queries completed")
print(f"Total time: {total_elapsed:.1f}s\n")
for complexity in ["SIMPLE", "MEDIUM", "COMPLEX"]:
successful = [r for r in results[complexity] if r.get('success')]
if successful:
latencies = [r['latency_ms'] for r in successful]
tokens = [r.get('tokens', 0) for r in successful]
print(f"{complexity}:")
print(f" Success rate: {len(successful)}/{len(results[complexity])}")
print(f" Latency (avg/min/max): {sum(latencies)/len(latencies):.0f}ms / {min(latencies):.0f}ms / {max(latencies):.0f}ms")
print(f" Tokens (avg): {sum(tokens)/len(tokens):.0f}")
else:
print(f"{complexity}: ALL FAILED")
# Save results
with open('baseline_benchmark_results.json', 'w') as f:
json.dump(results, f, indent=2)
print(f"\nResults saved to baseline_benchmark_results.json")
return results
if __name__ == "__main__":
benchmark_queries()
|