\u63a8\u7406\u6548\u7387\u6307\u6a19<\/strong><\/h2>\n

\u5ef6\u9072\u8207\u541e\u5410\u91cf<\/h3>\n

\u4f60\u9700\u8981\u8861\u91cf\u5ef6\u9072\u8207\u541e\u5410\u91cf\uff0c\u4f86\u4e86\u89e3 AI \u5de5\u4f5c\u8ca0\u8f09\u5728\u7f8e\u570b\u4f3a\u670d\u5668\u4e0a\u7684\u8868\u73fe\u3002\u5ef6\u9072\u8868\u793a\u6a21\u578b\u5c0d\u4f7f\u7528\u8005\u8acb\u6c42\u505a\u51fa\u56de\u61c9\u6240\u9700\u7684\u6642\u9593\u3002\u5c24\u5176\u5728\u5373\u6642\u61c9\u7528\u4e2d\uff0c\u5982\u679c\u4f60\u6709\u56b4\u683c\u7684\u5ef6\u9072\u8981\u6c42\uff0c\u5c31\u61c9\u540c\u6642\u95dc\u6ce8\u7aef\u5230\u7aef\u5ef6\u9072\u8207\u6a21\u578b\u5ef6\u9072\u3002\u5728\u505a\u4efb\u4f55\u8b8a\u66f4\u4e4b\u524d\uff0c\u5148\u5efa\u7acb\u63a8\u7406\u5ef6\u9072\u7684\u57fa\u7dda\u3002\u4f60\u53ef\u4ee5\u900f\u904e\u4f7f\u7528\u8f03\u5c0f\u7684\u6a21\u578b\u6216\u91cf\u5316\u6a21\u578b\u4f86\u6539\u5584\u6a21\u578b\u5ef6\u9072\u3002\u91cd\u9ede\u6700\u4f73\u5316\u9810\u586b\u968e\u6bb5\uff08prefill\uff09\uff0c\u4e26\u4f7f\u7528\u9ad8\u6548\u8def\u7531\uff0c\u4ee5\u964d\u4f4e\u9996 token \u6642\u9593\uff08time to first token\uff09\u3002\u541e\u5410\u91cf\u5247\u8861\u91cf\u7cfb\u7d71\u6bcf\u79d2\u53ef\u4ee5\u8655\u7406\u591a\u5c11 token \u6216\u8acb\u6c42\u3002token \u541e\u5410\u91cf\u4ee3\u8868\u6bcf\u5f35 GPU \u6bcf\u79d2\u7522\u751f\u7684 token \u6578\uff0c\u6709\u52a9\u65bc\u4f60\u4e86\u89e3 AI \u7cfb\u7d71\u7684\u6574\u9ad4\u541e\u5410\u80fd\u529b\u3002<\/p>\n

\u63d0\u793a\uff1a\u900f\u904e\u8abf\u6574\u6279\u5927\u5c0f\uff0c\u5728\u4e0d\u986f\u8457\u589e\u52a0\u5ef6\u9072\u7684\u524d\u63d0\u4e0b\u6700\u5927\u5316\u541e\u5410\u91cf\u3002<\/p><\/blockquote>\n

\u8cc7\u6e90\u4f7f\u7528\u7387<\/h3>\n

\u8cc7\u6e90\u4f7f\u7528\u7387\u53cd\u6620\u4e86 AI \u6a21\u578b\u5728\u786c\u9ad4\u8207\u80fd\u6e90\u4f7f\u7528\u4e0a\u7684\u6548\u7387\u3002\u4f60\u53ef\u4ee5\u4f7f\u7528\u57fa\u6e96\u6e2c\u8a66\u6846\u67b6\u4f86\u5ea6\u91cf\u6bcf\u500b\u8acb\u6c42\u7684\u5ef6\u9072\u3001\u541e\u5410\u91cf\u8207 GPU \u529f\u8017\u3002\u9019\u4e9b\u6846\u67b6\u9084\u6703\u7d0d\u5165\u96fb\u6e90\u4f7f\u7528\u6548\u7387\uff08PUE\uff09\u3001\u6c34\u8cc7\u6e90\u4f7f\u7528\u6548\u7387\uff08WUE\uff09\u4ee5\u53ca\u78b3\u5f37\u5ea6\u56e0\u5b50\uff08CIF\uff09\u7b49\u6307\u6a19\u3002\u5de5\u4f5c\u8ca0\u8f09\u80fd\u529b\u901a\u5e38\u4ee5\u6bcf\u79d2\u6d6e\u9ede\u904b\u7b97\u6b21\u6578\uff08FLOPS\uff09\u8861\u91cf\uff0c\u7528\u4f86\u8868\u793a\u4f3a\u670d\u5668\u7684\u904b\u7b97\u80fd\u529b\u3002AI \u4f3a\u670d\u5668\u53ef\u4ee5\u9054\u5230\u5146\u5104\u7d1a FLOPS\uff0c\u4f46\u5be6\u969b\u8868\u73fe\u4ecd\u53d6\u6c7a\u65bc\u786c\u9ad4\u6700\u4f73\u5316\u8207\u57fa\u790e\u8a2d\u65bd\u72c0\u6cc1\u3002\u4e0b\u8868\u5c55\u793a\u4e0d\u540c\u56e0\u7d20\u5c0d\u63a8\u7406\u6548\u7387\u7684\u5f71\u97ff\uff1a<\/p>\n

\n\n\n\n<\/colgroup>\n\n\n\n\n\n\n

\u56e0\u7d20<\/th>\n	\u5c0d\u63a8\u7406\u6548\u7387\u7684\u5f71\u97ff<\/th>\n<\/tr>\n
\u57fa\u790e\u8a2d\u65bd<\/td>\n	\u662f AI \u63a8\u7406\u6c38\u7e8c\u6027\u7684\u95dc\u9375\u6c7a\u5b9a\u56e0\u7d20\u3002<\/td>\n<\/tr>\n
\u786c\u9ad4\u6700\u4f73\u5316<\/td>\n	\u5be6\u969b\u6210\u679c\u6703\u56e0\u90e8\u7f72\u689d\u4ef6\u8207\u786c\u9ad4\u6548\u7387\u800c\u7522\u751f\u5dee\u7570\u3002<\/td>\n<\/tr>\n
\u80fd\u6e90\u4f7f\u7528<\/td>\n	\u6a21\u578b\u8a2d\u8a08\u53ef\u63d0\u5347\u7406\u8ad6\u6548\u7387\uff0c\u4f46\u771f\u5be6\u80fd\u8017\u4ecd\u6703\u6709\u6240\u4e0d\u540c\u3002<\/td>\n<\/tr>\n
\u8cc7\u6599\u4e2d\u5fc3\u4f4e\u6548\u80fd<\/td>\n	\u9ad8\u8017\u6c34\u8db3\u8de1\u5f80\u5f80\u6e90\u81ea\u8cc7\u6599\u4e2d\u5fc3\u6548\u7387\u4e0d\u4f73\uff0c\u800c\u4e0d\u50c5\u662f\u6a21\u578b\u672c\u8eab\u7279\u6027\u3002<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n \u6e96\u78ba\u6027\u8207\u4e00\u81f4\u6027<\/h3>\n \u4f60\u4e5f\u5fc5\u9808\u6aa2\u67e5\u6e96\u78ba\u6027\u8207\u4e00\u81f4\u6027\uff0c\u4ee5\u78ba\u4fdd AI \u63a8\u7406\u53ef\u9760\u3002\u7531 MLCommons \u5efa\u7acb\u7684 MLPerf \u57fa\u6e96\uff0c\u70ba\u786c\u9ad4\u3001\u8edf\u9ad4\u8207\u670d\u52d9\u7684\u8a13\u7df4\u8207\u63a8\u7406\u6548\u80fd\u63d0\u4f9b\u5ba2\u89c0\u8a55\u4f30\u3002\u9019\u4e9b\u57fa\u6e96\u5728\u56b4\u8b39\u689d\u4ef6\u4e0b\uff0c\u6e2c\u8a66\u5305\u62ec\u6df1\u5ea6\u5b78\u7fd2\u8207\u6a5f\u5668\u5b78\u7fd2\u5728\u5167\u7684\u591a\u7a2e\u6a21\u578b\u67b6\u69cb\u3002\u4e00\u81f4\u6027\u4ee3\u8868\u6a21\u578b\u5728\u591a\u6b21\u57f7\u884c\u4e2d\u6703\u7d66\u51fa\u76f8\u540c\u6216\u9ad8\u5ea6\u76f8\u4f3c\u7684\u7d50\u679c\u3002\u4e0b\u5716\uff08\u5047\u5b9a\u5b58\u5728\uff09\u5c0d\u6bd4\u4e86\u4e0d\u540c\u5f8c\u7aef\u8207\u57f7\u884c\u6a21\u5f0f\u4e0b\u63a8\u7406\u7d50\u679c\u4e0d\u4e00\u81f4\u7684\u60c5\u6cc1\uff1a<\/p>\n \u6e2c\u8a66\u74b0\u5883\u5efa\u7f6e<\/strong><\/h2>\n \u786c\u9ad4\u8207\u7db2\u8def<\/h3>\n \u8981\u5728\u7f8e\u570b\u4f3a\u670d\u5668\u4e0a\u7372\u5f97\u9ad8\u6548\u7684 AI \u63a8\u7406\uff0c\u4f60\u9700\u8981\u5408\u7406\u7684\u786c\u9ad4\u8207\u7db2\u8def\u914d\u7f6e\u3002\u5c0d\u591a\u6578\u751f\u7522\u90e8\u7f72\u800c\u8a00\uff0c\u4e00\u53f0\u914d\u5099 4 \u5f35 GPU \u7684 2U \u4f3a\u670d\u5668\u901a\u5e38\u8db3\u4ee5\u61c9\u4ed8\u3002\u5982\u679c\u4f60\u8ffd\u6c42\u6700\u5927\u541e\u5410\u91cf\u8207\u66f4\u4f73\u6563\u71b1\u8868\u73fe\uff0c4U \u4f3a\u670d\u5668\u6703\u662f\u66f4\u4f73\u9078\u64c7\u3002\u914d\u5099 96 GB \u986f\u793a\u8a18\u61b6\u9ad4\u7684 RTX PRO 6000 Blackwell GPU \u9069\u5408\u5c0f\u578b\u5718\u968a\uff0c\u800c\u66f4\u5927\u898f\u6a21\u90e8\u7f72\u5247\u53ef\u63a1\u7528 4 \u5f35\u6216 8 \u5f35 GPU \u7684\u7d44\u5408\u3002AMD EPYC 9005 \u8655\u7406\u5668\u64c1\u6709\u5927\u91cf PCIe Gen 5 \u901a\u9053\u8207\u5f37\u5927\u8a18\u61b6\u9ad4\u983b\u5bec\uff0c\u975e\u5e38\u9069\u5408\u591a GPU \u63a8\u7406\u60c5\u5883\u3002AI \u61c9\u7528\u5c0d\u7db2\u8def\u6548\u80fd\u4e5f\u6709\u5f88\u9ad8\u8981\u6c42\u3002\u5c0d\u65bc\u5373\u6642\u5f71\u97f3\u5206\u6790\u8207\u6df1\u5ea6\u5b78\u7fd2\u6a21\u578b\u8a13\u7df4\u800c\u8a00\uff0c\u9ad8\u983b\u5bec\u81f3\u95dc\u91cd\u8981\u3002\u5982\u679c\u983b\u5bec\u4e0d\u8db3\uff0c\u4f60\u53ef\u80fd\u6703\u9047\u5230\u8a13\u7df4\u6642\u9593\u62c9\u9577\u6216\u5373\u6642\u6d41\u7a0b\u5931\u6557\u7684\u554f\u984c\u3002\u5927\u578b AI \u6a21\u578b\u540c\u6a23\u9700\u8981\u53ef\u89c0\u7684\u8a18\u61b6\u9ad4\u983b\u5bec\uff0c\u4ee5\u5be6\u73fe\u5feb\u901f\u8cc7\u6599\u50b3\u8f38\u3002\u7576\u4f60\u589e\u52a0 GPU \u6578\u91cf\u6642\uff0c\u7cfb\u7d71\u5ef6\u9072\u6709\u6a5f\u6703\u964d\u4f4e\uff0c\u4f46\u524d\u63d0\u662f\u7db2\u8def\u983b\u5bec\u8db3\u4ee5\u652f\u6490\u3002<\/p>\n \u63d0\u793a\uff1a\u5728\u64f4\u5c55 AI \u5de5\u4f5c\u8ca0\u8f09\u4e4b\u524d\uff0c\u52d9\u5fc5\u5148\u6aa2\u67e5\u4e26\u78ba\u8a8d\u4f60\u7684\u7db2\u8def\u983b\u5bec\u662f\u5426\u5145\u8db3\u3002<\/p><\/blockquote>\n \u8edf\u9ad4\u3001\u6846\u67b6\u8207 Amazon SageMaker<\/h3>\n \u5728 AWS \u4e0a\u9032\u884c\u53ef\u64f4\u5c55\u7684 AI \u63a8\u7406\uff0c\u5efa\u8b70\u4f7f\u7528 SageMaker\u3002SageMaker \u8207 NVIDIA Triton Inference Server \u6df1\u5ea6\u6574\u5408\uff0c\u53ef\u5728\u6a21\u578b\u670d\u52d9\u4e2d\u6700\u5927\u5316\u541e\u5410\u91cf\u3001\u6700\u5c0f\u5316\u5ef6\u9072\u3002SageMaker \u652f\u63f4\u5305\u62ec TensorRT\u3001TensorFlow\u3001PyTorch \u8207 ONNX \u5728\u5167\u7684\u4e3b\u6d41 AI \u6846\u67b6\u3002SageMaker Inference Recommender \u80fd\u5354\u52a9\u4f60\u9078\u64c7\u5408\u9069\u7684\u904b\u7b97\u57f7\u884c\u500b\u9ad4\u985e\u578b\u3001\u57f7\u884c\u500b\u9ad4\u6578\u91cf\u8207\u63a8\u7406\u6700\u4f73\u5316\u65b9\u5f0f\uff0c\u5f9e\u800c\u514d\u53bb\u5927\u91cf\u7e41\u7463\u5de5\u4f5c\uff0c\u8b93\u4f60\u53ef\u4ee5\u66f4\u5feb\u5730\u9032\u884c\u5be6\u9a57\u8207\u8abf\u6821\u3002\u591a\u57f7\u884c\u7dd2\u53ef\u4ee5\u986f\u8457\u63d0\u5347\u541e\u5410\u91cf\uff0c\u5c24\u5176\u662f\u5728 GPU \u4e0a\u90e8\u7f72 Transformer \u6a21\u578b\u6642\u3002\u6279\u6b21\u8655\u7406\uff08batching\uff09\u5247\u662f\u53e6\u4e00\u9805\u95dc\u9375\u6280\u8853\u3002SageMaker \u8207 NVIDIA Triton Inference Server \u90fd\u652f\u63f4\u6279\u6b21\u8655\u7406\uff0c\u900f\u904e\u6279\u6b21\u8655\u7406\u53ef\u4ee5\u63d0\u5347\u541e\u5410\u91cf\u4e26\u964d\u4f4e\u5ef6\u9072\u3002MLPerf Inference \u70ba\u4f60\u7684 AI \u5de5\u4f5c\u8ca0\u8f09\u63d0\u4f9b\u6a19\u6e96\u5316\u57fa\u6e96\u3002\u4f60\u53ef\u4ee5\u4f7f\u7528 SageMaker Inference Recommender \u6e2c\u8a66\u4e0d\u540c\u914d\u7f6e\uff0c\u70ba\u4f60\u7684\u5373\u6642\u7aef\u9ede\u627e\u5230\u6700\u4f73\u65b9\u6848\u3002<\/p>\n \n\n\n\n<\/colgroup>\n\n\n\n\n\n\n \u7279\u6027<\/th>\n \u512a\u9ede<\/th>\n<\/tr>\n SageMaker + NVIDIA Triton<\/td>\n \u9ad8\u541e\u5410\u91cf\u3001\u4f4e\u5ef6\u9072\u7684\u6a21\u578b\u670d\u52d9\u80fd\u529b<\/td>\n<\/tr>\n SageMaker Inference Recommender<\/td>\n \u81ea\u52d5\u9078\u64c7\u57f7\u884c\u500b\u9ad4\u985e\u578b\u8207\u6700\u4f73\u5316\u7b56\u7565<\/td>\n<\/tr>\n \u591a\u57f7\u884c\u7dd2\u8207\u6279\u6b21\u8655\u7406<\/td>\n \u63d0\u5347\u541e\u5410\u91cf\u4e26\u964d\u4f4e\u5ef6\u9072<\/td>\n<\/tr>\n MLPerf Inference \u57fa\u6e96<\/td>\n \u6a19\u6e96\u5316\u7684\u6548\u80fd\u91cf\u6e2c\u65b9\u5f0f<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n \u8cc7\u6599\u6e96\u5099<\/h3>\n \u70ba\u4e86\u7372\u5f97\u6e96\u78ba\u7684\u63a8\u7406\u57fa\u6e96\u6e2c\u8a66\u7d50\u679c\uff0c\u4f60\u5fc5\u9808\u8b39\u614e\u6e96\u5099\u8cc7\u6599\u3002\u7cfb\u7d71\u5316\u7684\u53c3\u6578\u9078\u64c7\uff0c\u6709\u52a9\u65bc\u4f60\u66f4\u6709\u6548\u7387\u5730\u6e2c\u8a66 AI \u6a21\u578b\u3002\u5c0d\u65bc\u5373\u6642\u670d\u52d9\uff0c\u53ef\u4ee5\u8a2d\u5b9a\u5ef6\u9072\u9580\u6abb\uff0c\u4f8b\u5982 LLM \u4e92\u52d5\u4e2d\u6bcf\u500b token 50 ms \u7684\u4e0a\u9650\u3002\u63a1\u7528\u5408\u6210\u8cc7\u6599\u8207\u771f\u5be6\u8cc7\u6599\u4e26\u884c\u7684\u65b9\u5f0f\uff0c\u8a55\u4f30\u6a21\u578b\u5728\u4e0d\u540c\u8cc7\u6599\u578b\u614b\u4e0a\u7684\u8868\u73fe\u3002\u826f\u597d\u7684\u5be6\u9a57\u8a2d\u8a08\u8207\u5408\u7406\u7684\u53d6\u6a23\u7b56\u7565\uff0c\u80fd\u5e6b\u52a9\u4f60\u6709\u6548\u63a2\u7d22\u5927\u91cf\u53c3\u6578\u7d44\u5408\u3002\u8cc7\u6599\u524d\u8655\u7406\u5728\u63a8\u7406\u6548\u7387\u4e2d\u4e5f\u626e\u6f14\u91cd\u8981\u89d2\u8272\u3002\u5982\u679c\u4f60\u5728 CPU \u4e0a\u9032\u884c\u524d\u8655\u7406\uff0c\u53ef\u80fd\u6703\u51fa\u73fe\u74f6\u9838\u3002\u50cf NVIDIA DALI \u9019\u985e\u5de5\u5177\u53ef\u4ee5\u5c07\u524d\u8655\u7406\u5de5\u4f5c\u5378\u8f09\u5230 GPU \u4e0a\uff0c\u5f9e\u800c\u964d\u4f4e\u5ef6\u9072\u3001\u63d0\u5347\u541e\u5410\u91cf\u3002\u7576\u4f60\u5c07 DALI \u8207 NVIDIA Triton Inference Server \u6574\u5408\u6642\uff0c\u53ef\u4ee5\u5728 GPU \u4e0a\u5b8c\u6210\u8cc7\u6599\u89e3\u78bc\u8207\u7e2e\u653e\uff0c\u76e1\u91cf\u6e1b\u5c11\u901a\u8a0a\u958b\u92b7\u3002SageMaker \u652f\u63f4\u9019\u4e9b\u5de5\u4f5c\u6d41\u7a0b\uff0c\u5354\u52a9\u4f60\u66f4\u8f15\u9b06\u5730\u6700\u4f73\u5316 AI \u6a21\u578b\u90e8\u7f72\u3002SageMaker Inference Recommender \u4e5f\u80fd\u5354\u52a9\u4f60\u6e2c\u8a66\u6279\u6b21\u7b56\u7565\u8207\u524d\u8655\u7406\u6d41\u7a0b\uff0c\u9032\u800c\u70ba\u7aef\u9ede\u627e\u5230\u6700\u4f73\u914d\u7f6e\u3002<\/p>\n \u6ce8\u610f\uff1a\u5728 AWS \u4e0a\u5be6\u73fe\u53ef\u9760\u7684 AI \u63a8\u7406\u7d50\u679c\uff0c\u96e2\u4e0d\u958b\u5408\u7406\u7684\u8cc7\u6599\u6e96\u5099\u8207\u6279\u6b21\u7b56\u7565\u3002<\/p><\/blockquote>\n \u57f7\u884c\u63a8\u7406\u57fa\u6e96\u6e2c\u8a66<\/strong><\/h2>\n MLPerf Inference \u8207\u5176\u4ed6\u5de5\u5177<\/h3>\n \u4f60\u53ef\u4ee5\u900f\u904e\u57f7\u884c MLPerf Inference \u57fa\u6e96\u6e2c\u8a66\uff0c\u4f86\u8861\u91cf\u7f8e\u570b\u4f3a\u670d\u5668\u4e0a\u7684\u63a8\u7406\u6548\u7387\u3002MLPerf Inference \u662f\u91dd\u5c0d AI\u3001\u6df1\u5ea6\u5b78\u7fd2\u8207\u6a5f\u5668\u5b78\u7fd2\u5de5\u4f5c\u8ca0\u8f09\u7684\u9818\u5148\u57fa\u6e96\u6e2c\u8a66\u5957\u4ef6\uff0c\u5354\u52a9\u4f60\u6bd4\u8f03\u4e0d\u540c\u786c\u9ad4\u8207\u8edf\u9ad4\u7d44\u5408\u7684\u6548\u80fd\u3002\u8981\u958b\u59cb\u6e2c\u8a66\uff0c\u4f60\u9700\u8981\u4f9d\u7167\u4e0b\u5217\u6b65\u9a5f\u6e96\u5099\u74b0\u5883\u4e26\u78ba\u4fdd\u7d50\u679c\u6e96\u78ba\uff1a<\/p>\n \n \u5b89\u88dd MLCommons CM \u6846\u67b6\uff0c\u8a72\u6846\u67b6\u63d0\u4f9b\u7528\u65bc AI \u57fa\u6e96\u6e2c\u8a66\u7684\u81ea\u52d5\u5316\u914d\u65b9\u3002<\/li>\n \u4f7f\u7528\u5716\u5f62\u4ecb\u9762\u7522\u751f CM \u6307\u4ee4\uff0c\u4ee5\u81ea\u8a02\u4e26\u57f7\u884c MLPerf Inference \u57fa\u6e96\u6e2c\u8a66\u3002<\/li>\n \u900f\u904e pip install tensorflow<\/code> \u8207 pip install tensorflow-io<\/code> \u5b89\u88dd TensorFlow \u4f5c\u70ba\u5f8c\u7aef\u3002<\/li>\n \u8a2d\u5b9a\u74b0\u5883\u8b8a\u6578\u4ee5\u555f\u7528 oneDNN\uff0c\u4e26\u914d\u7f6e 16 \u4f4d\u6d6e\u9ede\u5132\u5b58\u683c\u5f0f\uff0c\u4ee5\u63d0\u5347\u6548\u7387\u3002<\/li>\n \u4f7f\u7528 wget -q https:\/\/zenodo.org\/record\/2535873\/files\/resnet50_v1.pb<\/code> \u4e0b\u8f09\u4f60\u8981\u6e2c\u8a66\u7684\u6a5f\u5668\u5b78\u7fd2\u6a21\u578b\uff0c\u4f8b\u5982 resnet50-v1.5<\/code>\u3002<\/li>\n \u4e0b\u8f09\u9069\u7528\u65bc\u8a72\u6a21\u578b\u7684\u8cc7\u6599\u96c6\uff0c\u4f8b\u5982 imagenet2012 \u9a57\u8b49\u96c6\u3002<\/li>\n \u4f7f\u7528\u6307\u4ee4 .\/run_local.sh tf resnet50 cpu<\/code> \u5728\u4f3a\u670d\u5668\u4e0a\u57f7\u884c\u57fa\u6e96\u6e2c\u8a66\u3002<\/li>\n<\/ol>\n \u4f60\u4e5f\u53ef\u4ee5\u4f7f\u7528\u5176\u4ed6\u5de5\u5177\uff0c\u4f8b\u5982\u652f\u63f4 TensorRT \u5f8c\u7aef\u7684 NVIDIA Triton Inference Server\uff0c\u7528\u65bc\u9ad8\u541e\u5410\u91cf AI \u5de5\u4f5c\u8ca0\u8f09\u3002Triton \u5167\u5efa\u6279\u6b21\u8655\u7406\u3001\u6a21\u578b\u7248\u672c\u7ba1\u7406\u4ee5\u53ca\u591a\u6846\u67b6\u90e8\u7f72\u7b49\u529f\u80fd\uff0c\u6709\u52a9\u65bc\u4f60\u6700\u4f73\u5316\u6a21\u578b\u5ef6\u9072\u8207\u6574\u9ad4\u541e\u5410\u91cf\u3002<\/p>\n \u63d0\u793a\uff1a\u52d9\u5fc5\u4f7f\u7528\u6a19\u6e96\u5316\u7684\u57fa\u6e96\u6e2c\u8a66\u5de5\u5177\uff0c\u78ba\u4fdd\u7d50\u679c\u516c\u5e73\u4e14\u53ef\u91cd\u8907\u3002<\/p><\/blockquote>\n \u57f7\u884c AI \u5de5\u4f5c\u8ca0\u8f09<\/h3>\n \u5728\u70ba\u63a8\u7406\u57fa\u6e96\u6e2c\u8a66\u57f7\u884c AI \u5de5\u4f5c\u8ca0\u8f09\u6642\uff0c\u4f60\u9700\u8981\u8a2d\u5b9a\u6e2c\u8a66\u60c5\u5883\uff0c\u4f7f\u5176\u5118\u53ef\u80fd\u8cbc\u8fd1\u771f\u5be6\u4f7f\u7528\u5834\u666f\u3002\u61c9\u540c\u6642\u95dc\u6ce8\u5373\u6642\u63a8\u7406\u8207\u6279\u6b21\u8655\u7406\u3002\u4e0b\u8868\u5c55\u793a\u5e38\u898b\u6e2c\u8a66\u985e\u578b\u3001\u95dc\u9375\u53c3\u6578\u4ee5\u53ca\u9700\u8981\u8ffd\u8e64\u7684\u6307\u6a19\uff1a<\/p>\n \n\n\n\n\n<\/colgroup>\n\n\n\n\n \u6e2c\u8a66\u985e\u578b<\/th>\n \u53c3\u6578<\/th>\n \u6307\u6a19<\/th>\n<\/tr>\n \u8a13\u7df4\u6e2c\u8a66<\/td>\n \u6279\u5927\u5c0f\u3001\u6a21\u578b\u5927\u5c0f<\/td>\n \u541e\u5410\u91cf\uff08\u6a23\u672c\/\u79d2\uff09\u3001GPU \u540d\u7a31<\/td>\n<\/tr>\n \u63a8\u7406\u6e2c\u8a66<\/td>\n \u6a21\u578b\u540d\u7a31\u3001\u6700\u5927\u4e26\u767c\u6578<\/td>\n \u6bcf\u79d2 token \u6578\u3001\u5e73\u5747\u5ef6\u9072\u3001GPU \u540d\u7a31<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n \u4f60\u53ef\u4ee5\u4f7f\u7528\u96a8\u6a5f\u53d6\u6a23\u7b49\u6280\u8853\uff0c\u70ba\u6e2c\u8a66\u6311\u9078\u5177\u4ee3\u8868\u6027\u7684\u8cc7\u6599\u9ede\u3002\u7dda\u6027\u6700\u4f73\u5316\u6709\u52a9\u65bc\u4f60\u627e\u5230\u6279\u6b21\u8655\u7406\u8207 TensorRT \u8a2d\u5b9a\u7684\u6700\u4f73\u7d44\u5408\uff1b\u8c9d\u6c0f\u63a8\u8ad6\u5247\u53ef\u5728\u8cc7\u6599\u6709\u9650\u7684\u60c5\u6cc1\u4e0b\u4f30\u8a08\u6548\u80fd\u8868\u73fe\uff0c\u7279\u5225\u9069\u7528\u65bc\u5b58\u5728\u4e0d\u78ba\u5b9a\u6027\u7684\u60c5\u5883\u3002\u9019\u4e9b\u65b9\u6cd5\u80fd\u5354\u52a9\u4f60\u5c07 AI \u5de5\u4f5c\u8ca0\u8f09\u8abf\u6821\u81f3\u66f4\u9ad8\u6548\u7387\u8207\u66f4\u4f73\u6548\u80fd\u3002<\/p>\n \u4f60\u4e5f\u9700\u8981\u91cd\u8996\u6279\u6b21\u8655\u7406\u7b56\u7565\u3002\u6279\u6b21\u8655\u7406\u900f\u904e\u5c07\u591a\u500b\u63a8\u7406\u8acb\u6c42\u5408\u4f75\u8655\u7406\uff0c\u4ee5\u63d0\u5347\u541e\u5410\u91cf\u3002NVIDIA Triton Inference Server \u8207 TensorRT \u90fd\u652f\u63f4\u52d5\u614b\u6279\u6b21\u8655\u7406\uff0c\u53ef\u4ee5\u6839\u64da\u8ca0\u8f09\u8b8a\u5316\u8abf\u6574\u6279\u5927\u5c0f\u3002\u5c0d\u65bc\u5373\u6642\u63a8\u7406\u800c\u8a00\uff0c\u9019\u7a2e\u65b9\u5f0f\u53ef\u5728\u964d\u4f4e\u6a21\u578b\u5ef6\u9072\u7684\u540c\u6642\u63d0\u5347\u6574\u9ad4\u541e\u5410\u91cf\u3002<\/p>\n \u6ce8\u610f\uff1a\u5408\u7406\u7684\u6279\u6b21\u8655\u7406\u7b56\u7565\u8207 TensorRT \u6700\u4f73\u5316\uff0c\u662f\u5be6\u73fe\u9ad8\u6548\u80fd AI \u63a8\u7406\u7684\u95dc\u9375\u3002<\/p><\/blockquote>\n \u8490\u96c6\u6548\u80fd\u8cc7\u6599<\/h3>\n \u5728\u57fa\u6e96\u6e2c\u8a66\u904e\u7a0b\u4e2d\uff0c\u4f60\u9700\u8981\u8490\u96c6\u8a73\u76e1\u7684\u6548\u80fd\u8cc7\u6599\u3002Perf Analyzer \u7b49\u5de5\u5177\u53ef\u4ee5\u5354\u52a9\u4f60\u76e3\u63a7\u4f3a\u670d\u5668\u7aef\u6307\u6a19\uff0c\u5305\u62ec GPU \u4f7f\u7528\u7387\u8207\u529f\u8017\u3002\u4f60\u53ef\u900f\u904e --collect-metrics<\/code> \u547d\u4ee4\u5217\u9078\u9805\u555f\u7528\u6307\u6a19\u8490\u96c6\u3002\u9810\u8a2d\u60c5\u6cc1\u4e0b\uff0c\u6307\u6a19\u6703\u66b4\u9732\u5728 localhost:8002\/metrics<\/code>\uff0c\u4f60\u4e5f\u53ef\u4ee5\u4f7f\u7528 --metrics-url<\/code> \u9078\u9805\u8b8a\u66f4\u6b64\u4f4d\u5740\u3002\u9810\u8a2d\u8490\u96c6\u9593\u9694\u70ba 1000 \u6beb\u79d2\uff0c\u53ef\u5229\u7528 --metrics-interval<\/code> \u53c3\u6578\u9032\u884c\u8abf\u6574\u3002<\/p>\n Perf Analyzer \u6703\u5728\u591a GPU \u7cfb\u7d71\u4e2d\u5f59\u7e3d\u6bcf\u5f35 GPU \u7684\u6307\u6a19\u3002\u4f60\u53ef\u4ee5\u5229\u7528 -f <filename><\/code> \u8207 --verbose-csv<\/code> \u9078\u9805\uff0c\u5c07\u7d50\u679c\u532f\u51fa\u70ba CSV \u6a94\u6848\uff0c\u65b9\u4fbf\u5f8c\u7e8c\u5206\u6790\u8da8\u52e2\u4e26\u8b58\u5225\u6548\u80fd\u74f6\u9838\u3002<\/p>\n \u5728\u9032\u884c\u8ca0\u8f09\u6e2c\u8a66\u6642\uff0c\u4f60\u61c9\u7576\u8b8a\u5316\u63a8\u7406\u8acb\u6c42\u7684\u901f\u7387\u8207\u5206\u4f48\u578b\u614b\uff0c\u56e0\u70ba\u9019\u4e9b\u56e0\u7d20\u5c0d\u5ef6\u9072\u8207\u541e\u5410\u91cf\u7684\u5f71\u97ff\u6975\u5927\u3002\u4f8b\u5982\uff1a<\/p>\n \n \u8d85\u5927\u898f\u6a21\u96f2\u7aef\u74b0\u5883\u7684\u57fa\u6e96\u6e2c\u8a66\u6703\u7279\u5225\u95dc\u6ce8\u6574\u9ad4\u53e2\u96c6\u6548\u7387\u8207\u670d\u52d9\u4e00\u81f4\u6027\u3002<\/li>\n \u5728\u5730\u4f01\u696d\u74b0\u5883\u66f4\u5f37\u8abf\u65bc\u53d7\u63a7\u57fa\u790e\u8a2d\u65bd\u5167\u7684\u6548\u80fd\u9a57\u8b49\u3002<\/li>\n \u908a\u7de3\u63a8\u7406\u60c5\u5883\u5247\u5c24\u5176\u95dc\u6ce8\u4f4e\u5ef6\u9072\u8207\u529f\u8017\u9650\u5236\u3002<\/li>\n \u5d4c\u5165\u5f0f\u6216\u5de5\u696d\u90e8\u7f72\u5247\u66f4\u91cd\u8996\u6c7a\u5b9a\u6027\u8207\u9577\u671f\u7a69\u5b9a\u904b\u4f5c\u3002<\/li>\n<\/ul>\n \u4f60\u53ef\u4ee5\u6839\u64da\u9019\u4e9b\u6d1e\u898b\uff0c\u91dd\u5c0d\u4e0d\u540c\u90e8\u7f72\u60c5\u5883\u8abf\u6574 TensorRT \u8207\u6279\u6b21\u8655\u7406\u8a2d\u5b9a\u3002\u6548\u80fd\u8abf\u6821\u9700\u8981\u4f60\u4ed4\u7d30\u5206\u6790\u8490\u96c6\u5230\u7684\u8cc7\u6599\uff0c\u4e26\u64da\u6b64\u505a\u51fa\u8abf\u6574\uff0c\u4ee5\u63d0\u5347\u63a8\u7406\u6548\u7387\u3002<\/p>\n \u63d0\u793a\uff1a\u5728\u8ca0\u8f09\u6e2c\u8a66\u4e2d\u52d9\u5fc5\u6301\u7e8c\u76e3\u63a7\u6a21\u578b\u5ef6\u9072\u3001\u541e\u5410\u91cf\u8207\u8cc7\u6e90\u4f7f\u7528\u60c5\u6cc1\uff0c\u78ba\u4fdd AI \u5de5\u4f5c\u8ca0\u8f09\u9054\u5230\u9810\u671f\u6548\u80fd\u76ee\u6a19\u3002<\/p><\/blockquote>\n \u7d50\u679c\u5206\u6790\u8207\u6700\u4f73\u5316<\/strong><\/h2>\n \u8b58\u5225\u74f6\u9838<\/h3>\n \u4f60\u9700\u8981\u5c0d\u57fa\u6e96\u6e2c\u8a66\u7d50\u679c\u9032\u884c\u6df1\u5165\u5206\u6790\uff0c\u4ee5\u627e\u51fa\u9650\u5236\u63a8\u7406\u6548\u7387\u7684\u74f6\u9838\u3002\u5e38\u898b\u554f\u984c\u5305\u62ec\u9ad8\u5ef6\u9072\u3001\u4f4e\u541e\u5410\u91cf\u4ee5\u53ca\u8a18\u61b6\u9ad4\u9650\u5236\u3002\u7576\u4f60\u57f7\u884c AI \u5de5\u4f5c\u8ca0\u8f09\u6642\uff0c\u53ef\u80fd\u6703\u767c\u73fe\u5927\u578b\u8a9e\u8a00\u6a21\u578b\u662f\u4ee5 token \u70ba\u55ae\u4f4d\u9010\u4e00\u751f\u6210\uff0c\u9019\u6703\u5728\u9577\u63d0\u793a\u8a5e\u60c5\u5883\u4e0b\u660e\u986f\u589e\u52a0\u6a21\u578b\u5ef6\u9072\u3002\u5728\u8acb\u6c42\u91cf\u5927\u3001\u5ef6\u9072\u9ad8\u7684\u60c5\u6cc1\u4e0b\uff0c\u4f3a\u670d\u5668\u53ef\u80fd\u51fa\u73fe\u7576\u6a5f\u6216\u903e\u6642\u3002\u6d41\u4e2d\u6279\u6b21\u8655\u7406\uff08inflight batching\uff09\u53ef\u4ee5\u6709\u6240\u52a9\u76ca\uff0c\u4f46\u540c\u6642\u4e5f\u5fc5\u9808\u5bc6\u5207\u7559\u610f\u8a18\u61b6\u9ad4\u9700\u6c42\u3002\u66f4\u5927\u7684\u6a21\u578b\u8207\u66f4\u9577\u7684\u63d0\u793a\u8a5e\u90fd\u6703\u6d88\u8017\u66f4\u591a\u986f\u793a\u8a18\u61b6\u9ad4\u3002\u96a8\u8457\u6a21\u578b\u8b8a\u5927\uff0c\u80fd\u8017\u4e5f\u6703\u4e0a\u5347\uff0c\u800c\u4f7f\u7528\u9ad8\u968e\u8a2d\u5099\u9032\u884c\u64f4\u5145\u5247\u6703\u5e36\u4f86\u66f4\u9ad8\u6210\u672c\u3002\u7ba1\u7406\u6a21\u578b\u66f4\u65b0\u8207\u7248\u672c\u8fed\u4ee3\u540c\u6a23\u95dc\u9375\uff0c\u6709\u52a9\u65bc\u5e73\u9806\u5b8c\u6210\u6a21\u578b\u90e8\u7f72\u3002<\/p>\n \n \u5ef6\u9072\uff1aLLM \u4ee5 token \u9010\u6b65\u751f\u6210\uff0c\u5c0e\u81f4\u9810\u6e2c\u6642\u9593\u62c9\u9577\u3002<\/li>\n \u541e\u5410\u91cf\uff1a\u9ad8\u8acb\u6c42\u91cf\u758a\u52a0\u9ad8\u5ef6\u9072\uff0c\u53ef\u80fd\u5c0e\u81f4\u4f3a\u670d\u5668\u7576\u6a5f\u3002<\/li>\n \u8a18\u61b6\u9ad4\uff1a\u66f4\u5927\u7684\u6a21\u578b\u8207\u66f4\u9577\u7684\u63d0\u793a\u8a5e\u9700\u8981\u66f4\u591a VRAM\u3002<\/li>\n \u80fd\u8017\uff1a\u5927\u578b\u6a21\u578b\u5c0d\u529f\u7387\u9700\u6c42\u66f4\u9ad8\u3002<\/li>\n \u53ef\u64f4\u5c55\u6027\uff1a\u9ad8\u968e\u6a5f\u5668\u5728\u64f4\u5145\u6642\u6210\u672c\u66f4\u9ad8\u3002<\/li>\n \u6a21\u578b\u66f4\u65b0\uff1a\u7248\u672c\u7ba1\u7406\u662f\u7a69\u5b9a\u90e8\u7f72\u7684\u95dc\u9375\u4e00\u6b65\u3002<\/li>\n<\/ul>\n \u63d0\u793a\uff1a\u5728\u5206\u6790\u74f6\u9838\u6642\uff0c\u8981\u7279\u5225\u95dc\u6ce8 GPU \u4f7f\u7528\u7387\u8207\u7db2\u8def\u57fa\u790e\u8a2d\u65bd\uff1b\u50c5\u4f9d\u8cf4\u50b3\u7d71 CPU \u6307\u6a19\uff0c\u5f80\u5f80\u7121\u6cd5\u771f\u5be6\u53cd\u6620 AI \u63a8\u7406\u6548\u80fd\u3002<\/p><\/blockquote>\n \u6700\u4f73\u5316\u63a8\u7406\u6548\u7387<\/h3>\n \u4f60\u53ef\u4ee5\u900f\u904e\u591a\u7a2e\u6700\u4f73\u5316\u7b56\u7565\u4f86\u63d0\u5347\u63a8\u7406\u6548\u7387\u3002\u91cf\u5316\uff08quantization\uff09\u900f\u904e\u964d\u4f4e\u6a21\u578b\u7cbe\u5ea6\uff0c\u6e1b\u5c11\u8a18\u61b6\u9ad4\u5360\u7528\u4e26\u52a0\u5feb\u63a8\u7406\u901f\u5ea6\u3002\u9023\u7e8c\u6279\u6b21\u8655\u7406\uff08continuous batching\uff09\u5247\u4ee5\u52d5\u614b\u65b9\u5f0f\u805a\u5408\u63a8\u7406\u8acb\u6c42\uff0c\u4ee5\u6700\u5927\u5316\u541e\u5410\u91cf\u3002KV \u5feb\u53d6\uff08KV caching\uff09\u53ef\u4ee5\u907f\u514d\u91cd\u8907\u904b\u7b97\uff0c\u52a0\u5feb token \u751f\u6210\u3002\u63d0\u524d\u7d50\u675f\u6a5f\u5236\uff08early exit\uff09\u5141\u8a31\u6a21\u578b\u5728\u4e0d\u5fc5\u5b8c\u6574\u901a\u904e\u6240\u6709\u5c64\u7684\u60c5\u6cc1\u4e0b\u63d0\u524d\u7d66\u51fa\u7d50\u679c\uff0c\u7bc0\u7701\u6642\u9593\u3002\u4f4e\u79e9\u5206\u89e3\uff08low-rank factorization\uff09\u5c07\u5927\u578b\u77e9\u9663\u5206\u89e3\u70ba\u8f03\u5c0f\u7684\u77e9\u9663\uff0c\u4ee5\u7372\u5f97\u66f4\u9ad8\u904b\u7b97\u901f\u5ea6\u8207\u66f4\u4f4e\u8a18\u61b6\u9ad4\u5360\u7528\u3002\u57fa\u790e\u8a2d\u65bd\u6700\u4f73\u5316\u5247\u900f\u904e\u6539\u826f\u7db2\u8def\u67b6\u69cb\uff0c\u964d\u4f4e\u8cc7\u6599\u50b3\u8f38\u6642\u9593\u3002\u5feb\u53d6\u8207\u5099\u5fd8\u9304\uff08caching vs. memoization\uff09\u6703\u5132\u5b58\u4e2d\u9593\u7d50\u679c\uff0c\u70ba\u5f8c\u7e8c\u8acb\u6c42\u63d0\u4f9b\u66f4\u5feb\u56de\u61c9\u3002\u900f\u904e\u5e73\u884c\u5316\u8207\u6279\u6b21\u8655\u7406\uff0c\u4f60\u53ef\u4ee5\u904b\u7528\u591a\u53f0\u4f3a\u670d\u5668\u8207\u591a\u5f35 GPU\uff0c\u4ee5\u7372\u5f97\u66f4\u4f73\u7684\u6574\u9ad4\u6548\u80fd\u3002<\/p>\n \n\n\n\n<\/colgroup>\n\n\n\n\n\n\n\n\n\n\n \u7b56\u7565<\/th>\n \u8aaa\u660e<\/th>\n<\/tr>\n \u91cf\u5316<\/td>\n \u964d\u4f4e\u6a21\u578b\u7cbe\u5ea6\uff0c\u4ee5\u6e1b\u5c11\u8a18\u61b6\u9ad4\u5360\u7528\u4e26\u52a0\u901f\u63a8\u7406\u3002<\/td>\n<\/tr>\n \u63d0\u524d\u7d50\u675f\u6a5f\u5236<\/td>\n \u5141\u8a31\u6a21\u578b\u5728\u4e26\u975e\u6240\u6709\u5c64\u5b8c\u6210\u904b\u7b97\u6642\u5c31\u7d66\u51fa\u9810\u6e2c\u7d50\u679c\u3002<\/td>\n<\/tr>\n \u4f4e\u79e9\u5206\u89e3<\/td>\n \u5c07\u5927\u578b\u77e9\u9663\u5206\u89e3\u70ba\u8f03\u5c0f\u77e9\u9663\uff0c\u4ee5\u7372\u5f97\u66f4\u9ad8\u901f\u5ea6\u8207\u66f4\u4f4e\u8a18\u61b6\u9ad4\u5360\u7528\u3002<\/td>\n<\/tr>\n \u9023\u7e8c\u6279\u6b21\u8655\u7406<\/td>\n \u4ee5\u52d5\u614b\u65b9\u5f0f\u805a\u5408\u8acb\u6c42\uff0c\u9032\u800c\u63d0\u5347\u541e\u5410\u91cf\u3002<\/td>\n<\/tr>\n KV \u5feb\u53d6<\/td>\n \u6e1b\u5c11\u91cd\u8907\u904b\u7b97\uff0c\u52a0\u5feb token \u751f\u6210\u901f\u5ea6\u3002<\/td>\n<\/tr>\n \u57fa\u790e\u8a2d\u65bd\u6700\u4f73\u5316<\/td>\n \u6539\u9032\u7db2\u8def\u67b6\u69cb\uff0c\u7e2e\u77ed\u8cc7\u6599\u50b3\u8f38\u6642\u9593\u3002<\/td>\n<\/tr>\n \u5feb\u53d6\u8207\u5099\u5fd8\u9304<\/td>\n \u5132\u5b58\u4e2d\u9593\u7d50\u679c\uff0c\u70ba\u5f8c\u7e8c\u63a8\u7406\u8acb\u6c42\u63d0\u4f9b\u66f4\u5feb\u901f\u56de\u61c9\u3002<\/td>\n<\/tr>\n \u5e73\u884c\u5316\u8207\u6279\u6b21\u8655\u7406<\/td>\n \u904b\u7528\u591a\u53f0\u4f3a\u670d\u5668\u8207\u591a\u5f35 GPU \u63d0\u5347\u6574\u9ad4\u6548\u80fd\u3002<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n \u4f60\u4e5f\u61c9\u7576\u5c0d\u591a\u57f7\u884c\u7dd2\u9032\u884c\u8abf\u6821\u3002\u5982\u679c\u57f7\u884c\u7dd2\u6578\u8a2d\u5b9a\u904e\u591a\uff0c\u53ef\u80fd\u5c0e\u81f4\u5ef6\u9072\u5347\u9ad8\u8207 GPU \u4f7f\u7528\u7387\u4e0b\u964d\u3002\u8b93\u57f7\u884c\u7dd2\u6578\u91cf\u8207\u4e26\u767c\u63a8\u7406\u8acb\u6c42\u6578\u66f4\u70ba\u5339\u914d\uff0c\u53ef\u4ee5\u540c\u6642\u63d0\u5347\u541e\u5410\u91cf\u4e26\u964d\u4f4e\u5ef6\u9072\u3002\u900f\u904e SageMaker Inference Recommender \u7b49\u5de5\u5177\u9032\u884c\u6548\u80fd\u8abf\u6821\uff0c\u53ef\u5354\u52a9\u4f60\u70ba AI \u5de5\u4f5c\u8ca0\u8f09\u627e\u5230\u6700\u5408\u9069\u7684\u6279\u6b21\u8655\u7406\u8207 TensorRT \u8a2d\u5b9a\u3002<\/p>\n \u904b\u7528 SageMaker \u9032\u884c AI \u64f4\u5c55<\/h3>\n Amazon SageMaker \u70ba\u4f60\u5728 AWS \u4e0a\u64f4\u5c55 AI \u63a8\u7406\u63d0\u4f9b\u5f37\u5927\u5de5\u5177\u3002\u4f60\u53ef\u4ee5\u5efa\u7acb\u5373\u6642\u63a8\u7406\u8207\u6279\u6b21\u63a8\u7406\u7aef\u9ede\uff0c\u6709\u6548\u7387\u5730\u7ba1\u7406\u6df1\u5ea6\u5b78\u7fd2\u8207\u6a5f\u5668\u5b78\u7fd2\u5de5\u4f5c\u8ca0\u8f09\u3002SageMaker \u652f\u63f4\u7121\u4f3a\u670d\u5668\u90e8\u7f72\uff08serverless\uff09\u8207\u81ea\u52d5\u64f4\u7e2e\u6a5f\u5236\uff0c\u5f9e\u800c\u5728\u7dad\u6301\u8cc7\u6e90\u5e73\u8861\u7684\u540c\u6642\u63a7\u5236\u6210\u672c\u3002\u4f60\u53ef\u4ee5\u900f\u904e Amazon CloudWatch \u76e3\u63a7\u6548\u80fd\uff0c\u4e26\u5feb\u901f\u6392\u9664\u554f\u984c\u3002SageMaker \u8ca0\u8cac\u8de8\u591a\u500b\u53ef\u7528\u5340\u7ba1\u7406\u6a21\u578b\u90e8\u7f72\uff0c\u4ee5\u78ba\u4fdd\u9ad8\u53ef\u9760\u6027\u3002\u591a\u6a21\u578b\u7aef\u9ede\uff08MMEs\uff09\u5141\u8a31\u4f60\u5728\u540c\u4e00\u500b\u7aef\u9ede\u4e0a\u90e8\u7f72\u591a\u500b\u6a21\u578b\uff0c\u9032\u800c\u6700\u5927\u5316 GPU \u4f7f\u7528\u7387\u4e26\u964d\u4f4e\u8a17\u7ba1\u6210\u672c\u3002SageMaker \u6703\u6839\u64da\u6d41\u91cf\u72c0\u6cc1\u52d5\u614b\u8f09\u5165\u8207\u5378\u8f09\u6a21\u578b\uff0c\u5728\u78ba\u4fdd\u5373\u6642\u63a8\u7406\u60c5\u5883\u4e2d\u4e9e\u79d2\u7d1a\u5ef6\u9072\u7684\u540c\u6642\u5be6\u73fe\u9ad8\u6548\u80fd\u3002SageMaker Inference Recommender \u80fd\u5354\u52a9\u4f60\u6e2c\u8a66\u4e0d\u540c\u914d\u7f6e\u3001\u6700\u4f73\u5316\u6279\u6b21\u8655\u7406\u4e26\u8abf\u6821 TensorRT \u8a2d\u5b9a\uff0c\u70ba\u5373\u6642\u7aef\u9ede\u627e\u5230\u6700\u4f73\u7d44\u5408\u3002\u4f60\u53ef\u4ee5\u5c07 SageMaker \u8207 NVIDIA Triton Inference Server \u642d\u914d\u4f7f\u7528\uff0c\u7372\u5f97\u9ad8\u541e\u5410\u91cf\u7684\u6a21\u578b\u670d\u52d9\u80fd\u529b\u3002SageMaker \u4e5f\u652f\u63f4\u6548\u80fd\u8abf\u6821\u3001\u8ca0\u8f09\u6e2c\u8a66\u8207\u57fa\u6e96\u6e2c\u8a66\uff0c\u4f7f\u4f60\u80fd\u5728 AWS \u4e0a\u8f15\u9b06\u6700\u4f73\u5316 AI \u5de5\u4f5c\u8ca0\u8f09\u3002<\/p>\n \n\n\n\n<\/colgroup>\n\n\n\n\n\n\n\n \u7279\u6027<\/th>\n \u8aaa\u660e<\/th>\n<\/tr>\n \u5373\u6642\u63a8\u7406\u8207\u6279\u6b21\u63a8\u7406<\/td>\n \u5efa\u7acb\u7aef\u9ede\u4ee5\u7372\u5f97\u5373\u6642\u9810\u6e2c\uff0c\u4e26\u6709\u6548\u7ba1\u7406\u5de5\u4f5c\u8ca0\u8f09\u3002<\/td>\n<\/tr>\n \u7121\u4f3a\u670d\u5668\u4e14\u5177\u6210\u672c\u512a\u52e2<\/td>\n \u900f\u904e\u81ea\u52d5\u64f4\u7e2e\u8207 AWS Lambda \u6574\u5408\u5be6\u73fe\u52d5\u614b\u8cc7\u6e90\u7ba1\u7406\u3002<\/td>\n<\/tr>\n \u76e3\u63a7\u8207\u9664\u932f<\/td>\n \u4f7f\u7528 Amazon CloudWatch \u9032\u884c\u5373\u6642\u76e3\u63a7\uff0c\u4e26\u5229\u7528\u9664\u932f\u5de5\u5177\u652f\u63f4\u6a21\u578b\u90e8\u7f72\u3002<\/td>\n<\/tr>\n \u90e8\u7f72\u80fd\u529b<\/td>\n \u904b\u7528\u591a\u7a2e\u57f7\u884c\u500b\u9ad4\u985e\u578b\u8207\u591a\u500b\u53ef\u7528\u5340\u64f4\u5c55\u57fa\u790e\u8a2d\u65bd\u3002<\/td>\n<\/tr>\n SageMaker \u591a\u6a21\u578b\u7aef\u9ede\uff08MMEs\uff09<\/td>\n \u5728\u55ae\u4e00\u7aef\u9ede\u4e0a\u90e8\u7f72\u591a\u500b\u6a21\u578b\uff0c\u5be6\u73fe\u5177\u6210\u672c\u512a\u52e2\u4e14\u53ef\u64f4\u5c55\u7684\u5373\u6642\u63a8\u7406\u3002<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n \u6ce8\u610f\uff1aSageMaker Inference Recommender \u662f\u6700\u4f73\u5316\u6a21\u578b\u90e8\u7f72\u3001\u6279\u6b21\u8655\u7406\u8207 TensorRT \u8a2d\u5b9a\u3001\u5be6\u73fe\u9802\u7d1a\u63a8\u7406\u6548\u7387\u7684\u95dc\u9375\u5de5\u5177\u3002<\/p><\/blockquote>\n \n <\/div>\n<\/div>\n \u900f\u904e\u9075\u5faa\u6e05\u6670\u7684\u6d41\u7a0b\uff0c\u4f60\u53ef\u4ee5\u5927\u5e45\u63d0\u5347\u7f8e\u570b\u4f3a\u670d\u5668\u4e0a\u7684\u63a8\u7406\u6548\u7387\u3002\u5148\u4f7f\u7528\u6a19\u6e96\u5316\u57fa\u6e96\u6846\u67b6\u91cf\u6e2c\u6548\u80fd\uff0c\u63a5\u8457\u904b\u7528 SageMaker \u8207 MLPerf \u53d6\u5f97\u6e96\u78ba\u7684\u63a8\u7406\u7d50\u679c\uff0c\u518d\u85c9\u7531 SageMaker Inference Recommender \u5b8c\u6210\u6a21\u578b\u90e8\u7f72\u8207\u6548\u80fd\u8abf\u6821\u3002\u6700\u5f8c\uff0c\u6301\u7e8c\u7d50\u5408\u5373\u6642\u63a8\u7406\u3001\u8ca0\u8f09\u6e2c\u8a66\u8207\u7aef\u9ede\u76e3\u63a7\uff0c\u4e0d\u65b7\u6700\u4f73\u5316\u4f60\u7684 AI \u5de5\u4f5c\u8ca0\u8f09\u3002<\/p>\n \u6301\u7e8c\u6700\u4f73\u5316\u5c0d\u65bc\u8cc7\u6e90\u7ba1\u7406\u3001\u6548\u80fd\u63d0\u5347\u4ee5\u53ca\u6a21\u578b\u76f8\u5bb9\u6027\u90fd\u81f3\u95dc\u91cd\u8981\u3002<\/p>\n \n\n\n\n\n<\/colgroup>\n\n\n\n\n\n \u7b56\u7565\u5c64\u7d1a<\/th>\n \u95dc\u6ce8\u9818\u57df<\/th>\n \u95dc\u9375\u884c\u52d5<\/th>\n<\/tr>\n 1<\/td>\n \u786c\u9ad4\u4f7f\u7528<\/td>\n \u4f7f\u7528\u6279\u6b21\u8655\u7406\u3001\u91cf\u5316\u8207 CUDA \u5716\u3002<\/td>\n<\/tr>\n 2<\/td>\n \u8a18\u61b6\u9ad4\u8207\u6f14\u7b97\u6cd5<\/td>\n \u6d88\u9664\u8a18\u61b6\u9ad4\u74f6\u9838\u3002<\/td>\n<\/tr>\n 3<\/td>\n \u64f4\u5c55\u62d3\u6a38<\/td>\n \u5c07\u4f3a\u670d\u5668\u5c31\u8fd1\u90e8\u7f72\u4e26\u59a5\u5584\u7ba1\u7406\u526f\u672c\u3002<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n \u7d50\u5408 SageMaker Inference Recommender\uff0c\u5728 AWS \u4e0a\u63a2\u7d22\u66f4\u9032\u968e\u7684 AI \u8207\u6df1\u5ea6\u5b78\u7fd2\u7b56\u7565\uff0c\u6301\u7e8c\u9032\u884c\u6548\u80fd\u8abf\u6821\uff0c\u4e26\u9ad8\u6548\u8655\u7406\u4e0d\u65b7\u6210\u9577\u7684\u63a8\u7406\u8acb\u6c42\u3002<\/p>\n \u5e38\u898b\u554f\u7b54<\/strong><\/h2>\n \u5728\u7f8e\u570b\u4f3a\u670d\u5668\u4e0a\u8861\u91cf\u63a8\u7406\u6548\u80fd\u7684\u6700\u4f73\u65b9\u5f0f\u662f\u4ec0\u9ebc\uff1f<\/h3>\n \u4f60\u61c9\u8a72\u4f7f\u7528 MLPerf Inference \u57fa\u6e96\u6e2c\u8a66\u3002\u9019\u4e9b\u6e2c\u8a66\u80fd\u70ba AI \u5de5\u4f5c\u8ca0\u8f09\u63d0\u4f9b\u6e05\u6670\u7684\u6548\u80fd\u7d50\u679c\u3002SageMaker \u53ef\u4ee5\u5354\u52a9\u4f60\u57f7\u884c\u9019\u4e9b\u57fa\u6e96\u6e2c\u8a66\uff0c\u4e26\u6bd4\u8f03\u4e0d\u540c\u786c\u9ad4\u7684\u8868\u73fe\u3002\u4f60\u53ef\u4ee5\u5728\u6bcf\u6b21\u6e2c\u8a66\u4e2d\u8ffd\u8e64\u5ef6\u9072\u3001\u541e\u5410\u91cf\u8207\u8cc7\u6e90\u4f7f\u7528\u60c5\u6cc1\u3002<\/p>\n SageMaker \u5982\u4f55\u63d0\u5347 AI \u63a8\u7406\u6548\u80fd\uff1f<\/h3>\n SageMaker \u8b93\u4f60\u53ef\u4ee5\u9ad8\u6548\u7387\u5730\u90e8\u7f72\u6a21\u578b\u3002\u4f60\u53ef\u4ee5\u904b\u7528\u6279\u6b21\u8655\u7406\u3001\u591a\u57f7\u884c\u7dd2\u4ee5\u53ca NVIDIA Triton \u7b49\u5de5\u5177\u4f86\u63d0\u5347\u63a8\u7406\u6548\u80fd\u3002SageMaker \u540c\u6642\u63d0\u4f9b\u76e3\u63a7\u8207\u8abf\u6821\u5de5\u5177\uff0c\u5354\u52a9\u4f60\u6301\u7e8c\u6700\u4f73\u5316 AI \u5de5\u4f5c\u8ca0\u8f09\uff0c\u7372\u5f97\u66f4\u4f73\u7d50\u679c\u3002<\/p>\n \u70ba\u4ec0\u9ebc\u6279\u6b21\u8655\u7406\u5c0d\u63a8\u7406\u6548\u80fd\u5f88\u91cd\u8981\uff1f<\/h3>\n \u6279\u6b21\u8655\u7406\u6703\u5c07\u591a\u500b\u8acb\u6c42\u5408\u4f75\u8655\u7406\uff0c\u9019\u7a2e\u65b9\u5f0f\u53ef\u4ee5\u986f\u8457\u63d0\u5347\u541e\u5410\u91cf\u4e26\u964d\u4f4e\u5ef6\u9072\u3002SageMaker \u8207 NVIDIA Triton \u90fd\u652f\u63f4\u52d5\u614b\u6279\u6b21\u8655\u7406\u3002\u53ea\u8981\u70ba\u4f60\u7684\u5de5\u4f5c\u8ca0\u8f09\u9078\u64c7\u5408\u9069\u7684\u6279\u5927\u5c0f\uff0c\u5c31\u80fd\u770b\u5230\u66f4\u597d\u7684 AI \u63a8\u7406\u8868\u73fe\u3002<\/p>\n \u5982\u4f55\u5728 SageMaker \u4e2d\u76e3\u63a7\u63a8\u7406\u6548\u80fd\uff1f<\/h3>\n \u4f60\u53ef\u4ee5\u5c07 Amazon CloudWatch \u8207 SageMaker \u7d50\u5408\u4f7f\u7528\u3002CloudWatch \u6703\u8ffd\u8e64\u5ef6\u9072\u3001\u541e\u5410\u91cf\u8207 GPU \u4f7f\u7528\u7387\u7b49\u6307\u6a19\u3002\u4f60\u4e5f\u53ef\u4ee5\u70ba\u6548\u80fd\u4e0b\u964d\u8a2d\u5b9a\u8b66\u793a\u3002SageMaker \u540c\u6642\u5141\u8a31\u4f60\u6e2c\u8a66\u4e0d\u540c\u914d\u7f6e\uff0c\u4ee5\u9032\u4e00\u6b65\u6539\u5584 AI \u63a8\u7406\u7d50\u679c\u3002<\/p>\n \u80fd\u5426\u900f\u904e SageMaker \u7aef\u9ede\u64f4\u5c55 AI \u63a8\u7406\uff1f<\/h3>\n \u53ef\u4ee5\u3002\u4f60\u53ef\u4ee5\u4f7f\u7528 SageMaker \u7aef\u9ede\u4f86\u64f4\u5c55\u63a8\u7406\u80fd\u529b\u3002SageMaker \u652f\u63f4\u81ea\u52d5\u64f4\u7e2e\u8207\u591a\u6a21\u578b\u7aef\u9ede\uff0c\u5f9e\u800c\u5728\u8655\u7406\u66f4\u591a\u8acb\u6c42\u7684\u540c\u6642\u7dad\u6301\u9ad8\u6548\u80fd\u3002\u96a8\u8457\u696d\u52d9\u9700\u6c42\u6210\u9577\uff0cSageMaker \u80fd\u5354\u52a9\u4f60\u8f15\u9b06\u7ba1\u7406 AI \u5de5\u4f5c\u8ca0\u8f09\u3002<\/p>\n","protected":false},"excerpt":{"rendered":" \u7576\u4f60\u5728\u7f8e\u570b\u4f3a\u670d\u5668\u4e0a\u90e8\u7f72 AI \u5de5\u4f5c\u8ca0\u8f09\u6642\uff0c\u9700\u8981\u5c0d\u63a8\u7406\u6548\u7387\u9032\u884c\u5ea6\u91cf\u3002MLPerf Inference \u7b49\u7522\u696d\u6a19\u6e96 […]<\/p>\n Read More…<\/a><\/p>\n","protected":false},"author":2,"featured_media":33334,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_acf_changed":false,"footnotes":""},"categories":[4815],"tags":[12193,12194,12195,12196,4883],"class_list":["post-33339","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-america-dedicated-server-tc","tag-inference-efficiency","tag-amazon-sagemaker","tag-mlperf-inference","tag-nvidia-triton-inference-server","tag-us-dedicated-server-tc"],"acf":[],"yoast_head":"\n