1026 lines
113 KiB
Plaintext
1026 lines
113 KiB
Plaintext
|
WARNING 12-10 19:52:40 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13.
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:52:40 [api_server.py:1772] vLLM API server version 0.12.0
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:52:40 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32768, 'max_num_seqs': 64}
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:52:45 [model.py:637] Resolved architecture: Qwen3ForCausalLM
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:52:45 [model.py:1750] Using max model len 32768
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:52:45 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:52:49 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>}, 'local_cache_dir': None}
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:52:50 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.3:34143 backend=nccl
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:52:50 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:52:50 [gpu_model_runner.py:3467] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic...
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:52:51 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:00, 9.44it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:00<00:00, 3.59it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:00<00:00, 2.78it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 2.48it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 2.80it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:52:53 [default_loader.py:308] Loading weights took 1.53 seconds
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:52:53 [gpu_model_runner.py:3549] Model loading took 15.3388 GiB memory and 2.474270 seconds
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:53:01 [backends.py:655] Using cache directory: /root/.cache/vllm/torch_compile_cache/a39c13ae20/rank_0_0/backbone for vLLM's torch.compile
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:53:01 [backends.py:715] Dynamo bytecode transform time: 7.49 s
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:53:02 [backends.py:257] Cache the graph for dynamic shape for later use
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:53:07 [backends.py:288] Compiling a graph for dynamic shape takes 5.35 s
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:53:14 [monitor.py:34] torch.compile takes 12.84 s in total
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:53:15 [gpu_worker.py:359] Available KV cache memory: 12.31 GiB
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:53:16 [kv_cache_utils.py:1286] GPU KV cache size: 80,672 tokens
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:53:16 [kv_cache_utils.py:1291] Maximum concurrency for 32,768 tokens per request: 2.46x
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 3/19 [00:00<00:00, 21.09it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 32%|███▏ | 6/19 [00:00<00:00, 21.75it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 47%|████▋ | 9/19 [00:00<00:00, 22.12it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 12/19 [00:00<00:00, 22.46it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 79%|███████▉ | 15/19 [00:00<00:00, 22.86it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 95%|█████████▍| 18/19 [00:00<00:00, 23.35it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:00<00:00, 22.82it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m
|
||
|
|
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 27%|██▋ | 3/11 [00:00<00:00, 21.12it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 55%|█████▍ | 6/11 [00:00<00:00, 22.80it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 82%|████████▏ | 9/11 [00:00<00:00, 23.92it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:00<00:00, 23.81it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:53:18 [gpu_model_runner.py:4466] Graph capturing finished in 2 secs, took 0.25 GiB
|
||
|
|
[0;36m(EngineCore_DP0 pid=4827)[0;0m INFO 12-10 19:53:18 [core.py:254] init engine (profile, create kv cache, warmup model) took 24.46 seconds
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:19 [api_server.py:1520] Supported tasks: ['generate']
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m WARNING 12-10 19:53:19 [model.py:1576] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:19 [serving_responses.py:194] Using default chat sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:19 [serving_chat.py:133] Using default chat sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [serving_completion.py:73] Using default completion sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [serving_chat.py:133] Using default chat sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [api_server.py:1847] Starting vLLM API server 0 on http://127.0.0.1:8000
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:38] Available routes are:
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /docs, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /health, Methods: GET
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /load, Methods: GET
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /pause, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /resume, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /is_paused, Methods: GET
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /tokenize, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /detokenize, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /v1/models, Methods: GET
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /version, Methods: GET
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /v1/audio/transcriptions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /v1/audio/translations, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /ping, Methods: GET
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /ping, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /invocations, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /metrics, Methods: GET
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /classify, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /v1/embeddings, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /score, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /v1/score, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /v1/rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /v2/rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:20 [launcher.py:46] Route: /pooling, Methods: POST
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: Started server process [4723]
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: Waiting for application startup.
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: Application startup complete.
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:46822 - "GET /v1/models HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:60990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:60990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32772 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:40 [loggers.py:236] Engine 000: Avg prompt throughput: 8.3 tokens/s, Avg generation throughput: 30.0 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.3%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:60990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40602 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40602 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40602 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:53:50 [loggers.py:236] Engine 000: Avg prompt throughput: 170.5 tokens/s, Avg generation throughput: 125.3 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51564 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51564 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:54:00 [loggers.py:236] Engine 000: Avg prompt throughput: 249.4 tokens/s, Avg generation throughput: 201.4 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40602 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32772 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:60990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:54:10 [loggers.py:236] Engine 000: Avg prompt throughput: 214.4 tokens/s, Avg generation throughput: 186.6 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32772 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40602 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:60990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32772 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32772 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:54:20 [loggers.py:236] Engine 000: Avg prompt throughput: 217.3 tokens/s, Avg generation throughput: 185.6 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51564 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32772 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57914 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32772 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:54:30 [loggers.py:236] Engine 000: Avg prompt throughput: 379.7 tokens/s, Avg generation throughput: 219.0 tokens/s, Running: 11 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.2%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:60990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40602 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57914 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51534 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32772 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40602 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57914 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:54:40 [loggers.py:236] Engine 000: Avg prompt throughput: 400.9 tokens/s, Avg generation throughput: 219.2 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57914 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40602 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51564 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40602 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:54:50 [loggers.py:236] Engine 000: Avg prompt throughput: 215.3 tokens/s, Avg generation throughput: 201.3 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.5%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:32776 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40602 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40602 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57036 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57060 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:60990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51564 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57060 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:60990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57060 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:55:00 [loggers.py:236] Engine 000: Avg prompt throughput: 298.4 tokens/s, Avg generation throughput: 234.4 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.6%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57036 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57036 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40602 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51886 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:55:10 [loggers.py:236] Engine 000: Avg prompt throughput: 301.7 tokens/s, Avg generation throughput: 316.4 tokens/s, Running: 13 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.4%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51564 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51564 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57060 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:60990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:55:20 [loggers.py:236] Engine 000: Avg prompt throughput: 160.7 tokens/s, Avg generation throughput: 304.3 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.1%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40602 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:55:30 [loggers.py:236] Engine 000: Avg prompt throughput: 36.4 tokens/s, Avg generation throughput: 211.4 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51886 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51886 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36574 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:55:40 [loggers.py:236] Engine 000: Avg prompt throughput: 278.6 tokens/s, Avg generation throughput: 206.7 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.1%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:60990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:60990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51886 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51886 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:55:50 [loggers.py:236] Engine 000: Avg prompt throughput: 172.8 tokens/s, Avg generation throughput: 236.4 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.1%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51886 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:57052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51550 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:56:00 [loggers.py:236] Engine 000: Avg prompt throughput: 157.6 tokens/s, Avg generation throughput: 255.1 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36574 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:56:10 [loggers.py:236] Engine 000: Avg prompt throughput: 65.8 tokens/s, Avg generation throughput: 130.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.2%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:40598 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:60990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49144 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49154 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49164 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49176 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49190 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49154 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:56:20 [loggers.py:236] Engine 000: Avg prompt throughput: 180.0 tokens/s, Avg generation throughput: 182.5 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49144 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49176 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:60990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49144 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36574 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49176 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49176 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36610 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36614 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:56:30 [loggers.py:236] Engine 000: Avg prompt throughput: 189.6 tokens/s, Avg generation throughput: 187.0 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.2%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36614 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49906 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49906 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49906 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:49164 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:60990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:56:40 [loggers.py:236] Engine 000: Avg prompt throughput: 139.6 tokens/s, Avg generation throughput: 259.9 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.1%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:56:50 [loggers.py:236] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 131.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:57:00 [loggers.py:236] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 17.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51020 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51026 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51068 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51068 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51068 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51094 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:57:10 [loggers.py:236] Engine 000: Avg prompt throughput: 323.0 tokens/s, Avg generation throughput: 108.9 tokens/s, Running: 12 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 7.4%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51068 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51094 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47876 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47888 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51026 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47914 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47930 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47930 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47958 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47930 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47970 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47888 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47978 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51094 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48006 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51094 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47888 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:57:20 [loggers.py:236] Engine 000: Avg prompt throughput: 1081.4 tokens/s, Avg generation throughput: 523.3 tokens/s, Running: 27 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.6%, Prefix cache hit rate: 25.8%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47914 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47888 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47914 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59078 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47888 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59128 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59078 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48006 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59128 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51026 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59078 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48006 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47958 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51068 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47970 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47978 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59128 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59128 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:57:30 [loggers.py:236] Engine 000: Avg prompt throughput: 1146.2 tokens/s, Avg generation throughput: 806.0 tokens/s, Running: 34 reqs, Waiting: 0 reqs, GPU KV cache usage: 17.4%, Prefix cache hit rate: 38.4%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59128 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47888 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47876 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59128 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47914 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51094 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51020 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47930 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59078 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47930 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48074 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47930 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47958 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:57:40 [loggers.py:236] Engine 000: Avg prompt throughput: 610.1 tokens/s, Avg generation throughput: 902.5 tokens/s, Running: 42 reqs, Waiting: 0 reqs, GPU KV cache usage: 17.6%, Prefix cache hit rate: 43.4%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48090 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47888 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48090 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48090 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47888 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48006 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51068 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47978 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47930 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51020 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51094 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:57:50 [loggers.py:236] Engine 000: Avg prompt throughput: 625.7 tokens/s, Avg generation throughput: 866.7 tokens/s, Running: 31 reqs, Waiting: 0 reqs, GPU KV cache usage: 15.5%, Prefix cache hit rate: 47.7%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47914 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47970 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47888 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48006 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51020 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51026 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51094 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47958 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48074 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48090 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47888 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47958 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51094 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59128 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51068 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47876 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48090 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47888 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59128 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:58:00 [loggers.py:236] Engine 000: Avg prompt throughput: 936.5 tokens/s, Avg generation throughput: 828.3 tokens/s, Running: 39 reqs, Waiting: 0 reqs, GPU KV cache usage: 19.6%, Prefix cache hit rate: 43.0%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51068 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48074 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48090 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59128 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50246 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50254 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50270 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50280 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50286 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50270 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50270 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50286 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50292 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50292 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47978 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47876 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59128 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:58:10 [loggers.py:236] Engine 000: Avg prompt throughput: 997.0 tokens/s, Avg generation throughput: 957.7 tokens/s, Running: 41 reqs, Waiting: 0 reqs, GPU KV cache usage: 18.7%, Prefix cache hit rate: 38.5%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47970 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47888 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47958 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47876 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59078 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51094 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47914 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48006 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47958 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48090 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50246 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51020 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50254 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:58:20 [loggers.py:236] Engine 000: Avg prompt throughput: 537.1 tokens/s, Avg generation throughput: 829.6 tokens/s, Running: 36 reqs, Waiting: 0 reqs, GPU KV cache usage: 14.6%, Prefix cache hit rate: 36.5%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48090 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50246 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50280 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51068 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50286 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51026 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50292 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47978 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50280 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50292 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50474 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50270 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50482 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50504 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48006 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50254 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50482 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50254 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50518 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50522 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50526 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50518 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50528 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:58:30 [loggers.py:236] Engine 000: Avg prompt throughput: 978.6 tokens/s, Avg generation throughput: 953.1 tokens/s, Running: 53 reqs, Waiting: 0 reqs, GPU KV cache usage: 19.2%, Prefix cache hit rate: 33.2%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50526 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47958 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51094 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48074 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50254 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47978 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51068 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59078 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48006 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51020 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51026 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47914 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:58:40 [loggers.py:236] Engine 000: Avg prompt throughput: 528.9 tokens/s, Avg generation throughput: 1091.1 tokens/s, Running: 39 reqs, Waiting: 0 reqs, GPU KV cache usage: 18.0%, Prefix cache hit rate: 31.7%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59078 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50292 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47970 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48090 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50482 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50270 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50254 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50522 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50474 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48090 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50286 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47978 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50246 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47958 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50528 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:58:50 [loggers.py:236] Engine 000: Avg prompt throughput: 963.7 tokens/s, Avg generation throughput: 833.9 tokens/s, Running: 31 reqs, Waiting: 0 reqs, GPU KV cache usage: 14.2%, Prefix cache hit rate: 29.3%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50518 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50292 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47958 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50504 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50522 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48090 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51026 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50482 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50528 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50504 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50246 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51068 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:59:00 [loggers.py:236] Engine 000: Avg prompt throughput: 877.4 tokens/s, Avg generation throughput: 721.8 tokens/s, Running: 28 reqs, Waiting: 0 reqs, GPU KV cache usage: 13.8%, Prefix cache hit rate: 27.4%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50474 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50254 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50246 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47978 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50528 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50522 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50504 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47970 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:59:10 [loggers.py:236] Engine 000: Avg prompt throughput: 506.8 tokens/s, Avg generation throughput: 718.1 tokens/s, Running: 25 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.1%, Prefix cache hit rate: 26.4%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50518 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50528 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50522 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50474 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50522 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:54110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50474 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50286 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50482 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50246 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50286 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50254 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50518 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50246 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50518 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:54112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:54118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50518 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50474 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:59:20 [loggers.py:236] Engine 000: Avg prompt throughput: 898.4 tokens/s, Avg generation throughput: 827.2 tokens/s, Running: 36 reqs, Waiting: 0 reqs, GPU KV cache usage: 13.4%, Prefix cache hit rate: 24.9%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:54110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50518 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47970 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51068 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50504 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50246 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50254 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:54118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50474 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47978 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:54112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50254 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50246 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:59:30 [loggers.py:236] Engine 000: Avg prompt throughput: 784.6 tokens/s, Avg generation throughput: 736.9 tokens/s, Running: 26 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.3%, Prefix cache hit rate: 23.7%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:54110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50518 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:54112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50522 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50286 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50504 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50522 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:54110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50518 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51068 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50522 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50504 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50474 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50254 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:59:40 [loggers.py:236] Engine 000: Avg prompt throughput: 786.4 tokens/s, Avg generation throughput: 757.8 tokens/s, Running: 33 reqs, Waiting: 0 reqs, GPU KV cache usage: 13.5%, Prefix cache hit rate: 22.5%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:54118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50522 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50246 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50528 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50254 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47970 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47990 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50254 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:54110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47978 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50482 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 19:59:50 [loggers.py:236] Engine 000: Avg prompt throughput: 556.2 tokens/s, Avg generation throughput: 802.9 tokens/s, Running: 32 reqs, Waiting: 0 reqs, GPU KV cache usage: 13.2%, Prefix cache hit rate: 21.8%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51068 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50522 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50246 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50246 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48038 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:53502 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47970 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50254 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50246 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:36416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:53516 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51122 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50504 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51082 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50518 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59112 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:53516 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50474 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50518 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO 12-10 20:00:00 [loggers.py:236] Engine 000: Avg prompt throughput: 1196.3 tokens/s, Avg generation throughput: 791.3 tokens/s, Running: 34 reqs, Waiting: 0 reqs, GPU KV cache usage: 14.5%, Prefix cache hit rate: 20.4%
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47862 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:47938 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50474 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51050 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:48052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:54118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:51136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50490 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50504 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=4723)[0;0m INFO: 127.0.0.1:50518 - "POST /v1/completions HTTP/1.1" 200 OK
|