1023 lines
112 KiB
Plaintext
1023 lines
112 KiB
Plaintext
|
WARNING 12-10 09:24:11 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13.
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:11 [api_server.py:1772] vLLM API server version 0.12.0
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:11 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 65536, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64}
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:21 [model.py:637] Resolved architecture: LlamaForCausalLM
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:21 [model.py:1750] Using max model len 65536
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:21 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:30 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=65536, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>}, 'local_cache_dir': None}
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:30 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:46573 backend=nccl
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:30 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:31 [gpu_model_runner.py:3467] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct...
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:32 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:00, 4.87it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:01<00:01, 1.62it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:02<00:00, 1.18it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:03<00:00, 1.08it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:03<00:00, 1.22it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:37 [default_loader.py:308] Loading weights took 3.50 seconds
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:38 [gpu_model_runner.py:3549] Model loading took 14.9889 GiB memory and 5.683476 seconds
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:44 [backends.py:655] Using cache directory: /root/.cache/vllm/torch_compile_cache/d7f4cb5e01/rank_0_0/backbone for vLLM's torch.compile
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:44 [backends.py:715] Dynamo bytecode transform time: 5.00 s
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:45 [backends.py:257] Cache the graph for dynamic shape for later use
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:49 [backends.py:288] Compiling a graph for dynamic shape takes 3.82 s
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:50 [monitor.py:34] torch.compile takes 8.82 s in total
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:52 [gpu_worker.py:359] Available KV cache memory: 21.99 GiB
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:52 [kv_cache_utils.py:1286] GPU KV cache size: 180,128 tokens
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:52 [kv_cache_utils.py:1291] Maximum concurrency for 65,536 tokens per request: 2.75x
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 11%|█ | 2/19 [00:00<00:00, 17.69it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 26%|██▋ | 5/19 [00:00<00:00, 19.57it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 42%|████▏ | 8/19 [00:00<00:00, 20.71it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 58%|█████▊ | 11/19 [00:00<00:00, 21.70it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 74%|███████▎ | 14/19 [00:00<00:00, 22.92it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 89%|████████▉ | 17/19 [00:00<00:00, 24.08it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:00<00:00, 22.20it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m
|
||
|
|
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 18%|█▊ | 2/11 [00:00<00:00, 19.47it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 45%|████▌ | 5/11 [00:00<00:00, 23.29it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 73%|███████▎ | 8/11 [00:00<00:00, 24.79it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:00<00:00, 25.49it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:00<00:00, 24.66it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:55 [gpu_model_runner.py:4466] Graph capturing finished in 3 secs, took 0.84 GiB
|
||
|
|
[0;36m(EngineCore_DP0 pid=3002)[0;0m INFO 12-10 09:24:55 [core.py:254] init engine (profile, create kv cache, warmup model) took 16.62 seconds
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:58 [api_server.py:1520] Supported tasks: ['generate']
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m WARNING 12-10 09:24:58 [model.py:1576] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:58 [serving_responses.py:194] Using default chat sampling params from model: {'temperature': 0.6, 'top_p': 0.9}
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [serving_chat.py:133] Using default chat sampling params from model: {'temperature': 0.6, 'top_p': 0.9}
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [serving_completion.py:73] Using default completion sampling params from model: {'temperature': 0.6, 'top_p': 0.9}
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [serving_chat.py:133] Using default chat sampling params from model: {'temperature': 0.6, 'top_p': 0.9}
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [api_server.py:1847] Starting vLLM API server 0 on http://127.0.0.1:8000
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:38] Available routes are:
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /openapi.json, Methods: HEAD, GET
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /docs, Methods: HEAD, GET
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: HEAD, GET
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /redoc, Methods: HEAD, GET
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /health, Methods: GET
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /load, Methods: GET
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /pause, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /resume, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /is_paused, Methods: GET
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /tokenize, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /detokenize, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /v1/models, Methods: GET
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /version, Methods: GET
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /v1/audio/transcriptions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /v1/audio/translations, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /ping, Methods: GET
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /ping, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /invocations, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /metrics, Methods: GET
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /classify, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /v1/embeddings, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /score, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /v1/score, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /v1/rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /v2/rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:24:59 [launcher.py:46] Route: /pooling, Methods: POST
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: Started server process [2740]
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: Waiting for application startup.
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: Application startup complete.
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48186 - "GET /v1/models HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55546 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55562 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:25:30 [loggers.py:236] Engine 000: Avg prompt throughput: 117.3 tokens/s, Avg generation throughput: 142.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.6%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55546 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55546 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55562 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49540 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:25:40 [loggers.py:236] Engine 000: Avg prompt throughput: 201.8 tokens/s, Avg generation throughput: 170.3 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.7%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55546 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:25:50 [loggers.py:236] Engine 000: Avg prompt throughput: 107.1 tokens/s, Avg generation throughput: 139.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55546 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55546 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55546 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:26:00 [loggers.py:236] Engine 000: Avg prompt throughput: 220.0 tokens/s, Avg generation throughput: 201.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.6%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:26:10 [loggers.py:236] Engine 000: Avg prompt throughput: 285.9 tokens/s, Avg generation throughput: 188.6 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.8%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:26:20 [loggers.py:236] Engine 000: Avg prompt throughput: 301.1 tokens/s, Avg generation throughput: 209.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.1%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:26:30 [loggers.py:236] Engine 000: Avg prompt throughput: 438.5 tokens/s, Avg generation throughput: 304.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44386 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44402 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:26:40 [loggers.py:236] Engine 000: Avg prompt throughput: 280.0 tokens/s, Avg generation throughput: 87.5 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44386 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44386 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:26:50 [loggers.py:236] Engine 000: Avg prompt throughput: 198.2 tokens/s, Avg generation throughput: 368.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.8%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44402 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44386 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44386 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44402 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:27:00 [loggers.py:236] Engine 000: Avg prompt throughput: 288.1 tokens/s, Avg generation throughput: 370.2 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.9%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:34818 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:27:10 [loggers.py:236] Engine 000: Avg prompt throughput: 159.8 tokens/s, Avg generation throughput: 125.2 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:34818 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:27:20 [loggers.py:236] Engine 000: Avg prompt throughput: 33.4 tokens/s, Avg generation throughput: 210.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.3%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:34818 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:34818 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:27:30 [loggers.py:236] Engine 000: Avg prompt throughput: 321.8 tokens/s, Avg generation throughput: 234.0 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:34818 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:27:40 [loggers.py:236] Engine 000: Avg prompt throughput: 187.7 tokens/s, Avg generation throughput: 279.1 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:57388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:34818 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:27:50 [loggers.py:236] Engine 000: Avg prompt throughput: 91.3 tokens/s, Avg generation throughput: 168.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.1%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53634 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53634 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53634 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53634 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:28:00 [loggers.py:236] Engine 000: Avg prompt throughput: 133.9 tokens/s, Avg generation throughput: 68.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.2%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51502 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51502 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51512 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51512 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:28:10 [loggers.py:236] Engine 000: Avg prompt throughput: 80.2 tokens/s, Avg generation throughput: 255.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.2%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51502 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53634 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51512 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53634 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53634 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:59742 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:59748 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:59748 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:59768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:28:20 [loggers.py:236] Engine 000: Avg prompt throughput: 330.3 tokens/s, Avg generation throughput: 233.8 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53634 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51502 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:59742 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:28:30 [loggers.py:236] Engine 000: Avg prompt throughput: 8.9 tokens/s, Avg generation throughput: 136.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:28:40 [loggers.py:236] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:28:50 [loggers.py:236] Engine 000: Avg prompt throughput: 217.3 tokens/s, Avg generation throughput: 138.8 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 5.1%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:29:00 [loggers.py:236] Engine 000: Avg prompt throughput: 977.3 tokens/s, Avg generation throughput: 702.9 tokens/s, Running: 13 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 23.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:29:10 [loggers.py:236] Engine 000: Avg prompt throughput: 1114.3 tokens/s, Avg generation throughput: 886.0 tokens/s, Running: 16 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 36.4%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:29:20 [loggers.py:236] Engine 000: Avg prompt throughput: 824.8 tokens/s, Avg generation throughput: 992.6 tokens/s, Running: 12 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 43.5%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:29:30 [loggers.py:236] Engine 000: Avg prompt throughput: 528.8 tokens/s, Avg generation throughput: 739.8 tokens/s, Running: 12 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 47.2%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:29:40 [loggers.py:236] Engine 000: Avg prompt throughput: 904.2 tokens/s, Avg generation throughput: 1004.7 tokens/s, Running: 18 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.1%, Prefix cache hit rate: 43.5%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:29:50 [loggers.py:236] Engine 000: Avg prompt throughput: 998.8 tokens/s, Avg generation throughput: 812.2 tokens/s, Running: 16 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 38.9%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:30:00 [loggers.py:236] Engine 000: Avg prompt throughput: 547.5 tokens/s, Avg generation throughput: 743.1 tokens/s, Running: 15 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 36.7%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53120 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53164 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:30:10 [loggers.py:236] Engine 000: Avg prompt throughput: 902.9 tokens/s, Avg generation throughput: 1072.2 tokens/s, Running: 24 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 33.6%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53120 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:30:20 [loggers.py:236] Engine 000: Avg prompt throughput: 541.0 tokens/s, Avg generation throughput: 1051.2 tokens/s, Running: 12 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 32.0%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53120 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53164 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:30:30 [loggers.py:236] Engine 000: Avg prompt throughput: 820.5 tokens/s, Avg generation throughput: 727.3 tokens/s, Running: 12 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 29.9%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53120 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32770 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:51390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60986 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:30:40 [loggers.py:236] Engine 000: Avg prompt throughput: 902.7 tokens/s, Avg generation throughput: 642.3 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 28.4%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:30:50 [loggers.py:236] Engine 000: Avg prompt throughput: 477.5 tokens/s, Avg generation throughput: 641.1 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 27.4%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:31:00 [loggers.py:236] Engine 000: Avg prompt throughput: 941.3 tokens/s, Avg generation throughput: 918.7 tokens/s, Running: 20 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 25.7%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:31:10 [loggers.py:236] Engine 000: Avg prompt throughput: 689.2 tokens/s, Avg generation throughput: 763.4 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, Prefix cache hit rate: 24.6%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:31:20 [loggers.py:236] Engine 000: Avg prompt throughput: 886.5 tokens/s, Avg generation throughput: 589.1 tokens/s, Running: 14 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 23.2%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:31:30 [loggers.py:236] Engine 000: Avg prompt throughput: 686.3 tokens/s, Avg generation throughput: 807.1 tokens/s, Running: 14 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 22.3%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55152 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55152 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53130 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53150 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60988 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:47670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO 12-10 09:31:40 [loggers.py:236] Engine 000: Avg prompt throughput: 906.9 tokens/s, Avg generation throughput: 878.0 tokens/s, Running: 13 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 21.1%
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48766 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:55152 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:49612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:45940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:60962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:39334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:50416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:53142 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=2740)[0;0m INFO: 127.0.0.1:48794 - "POST /v1/completions HTTP/1.1" 200 OK
|