GPU\u6545\u969c\u7684\u8868\u73fe\u5f62\u5f0f\u901a\u5e38\u8207\u5176\u6839\u6e90\u76f8\u95dc\uff1a<\/p>\n

\n
• \u9a45\u52d5\u7248\u672c\u4e0d\u5339\u914d<\/strong>\u9999\u6e2f\u4f3a\u670d\u5668\u4f48\u7f72\u4e2d\uff0c\u6838\u5fc3\u66f4\u65b0\u5f8c\u9a45\u52d5\u53ef\u80fd\u51fa\u73fe\u4e0d\u76f8\u5bb9\u3002\u900f\u904euname -r<\/code>\u67e5\u770b\u6838\u5fc3\u7248\u672c\uff0c\u4e26\u8207NVIDIA\u9a45\u52d5\u767c\u5e03\u8aaa\u660e\u6bd4\u5c0d\uff0c\u78ba\u8a8d\u76f8\u5bb9\u6027\u3002<\/li>\n
• \u8cc7\u6e90\u904e\u5ea6\u5206\u914d<\/strong>\u5728\u5bb9\u5668\u5316\u74b0\u5883\u4e2d\uff0cnvidia-container-cli stats<\/code>\u7b49\u5de5\u5177\u53ef\u67e5\u770bDocker\/Kubernetes\u5bb9\u5668\u662f\u5426\u8d85\u51fa\u5206\u914d\u7684GPU\u8a18\u61b6\u9ad4\u2014\u2014\u9019\u662f\u5171\u7528\u4f3a\u670d\u5668\u79df\u7528\u5834\u666f\u4e2d\u7684\u5e38\u898b\u554f\u984c\u3002<\/li>\n
• \u786c\u9ad4\u8001\u5316<\/strong>\u9999\u6e2f\u6eab\u6696\u6c23\u5019\u4e0b\uff0cGPU\u53cd\u8986\u71b1\u7bc0\u6d41\u53ef\u80fd\u5c0e\u81f4\u6c38\u4e45\u6027\u640d\u58de\u3002\u57f7\u884cnvidia-smi -f \/tmp\/gpu_stats.log -l 60<\/code>\uff0c\u53ef\u64f7\u53d6\u9577\u671f\u6548\u80fd\u8870\u6e1b\u8da8\u52e2\u3002<\/li>\n<\/ul>\n

  [\/vc_column_text][\/vc_column][\/vc_row][vc_row el_class=”blog-detail-section”][vc_column][vc_column_text css=””]<\/p>\n

  \u9999\u6e2f\u4f3a\u670d\u5668\u57fa\u790e\u67b6\u69cb\u4e2dGPU\u6545\u969c\u7684\u5be6\u7528\u4fee\u5fa9\u65b9\u6848<\/strong><\/h2>\n

  1. \u9a45\u52d5\u7ba1\u7406\u8207\u8edf\u9ad4\u4fee\u5fa9<\/h3>\n

  \u66f4\u65b0\u6216\u91cd\u88dd\u9a45\u52d5\u9700\u9075\u5faa\u7cfb\u7d71\u5316\u6d41\u7a0b\uff1a<\/p>\n

  \n
  1. \u5fb9\u5e95\u89e3\u9664\u5b89\u88dd\u820a\u9a45\u52d5<\/strong>\n

     apt-get remove --purge nvidia-*\r\nrm -rf \/etc\/nvidia \/usr\/lib\/nvidia<\/code><\/pre>\n<\/li>\n
     \u7248\u672c\u5339\u914d\u5b89\u88dd<\/strong>\u5f9e\u5b98\u65b9\u5132\u5b58\u5eab\u4e0b\u8f09\u9a45\u52d5\uff0c\u78ba\u4fdd\u8207Linux\u6838\u5fc3\u53ca\u4f3a\u670d\u5668\u67b6\u69cb\u5339\u914d\u3002\u91dd\u5c0d\u9999\u6e2f\u8cc7\u6599\u4e2d\u5fc3\uff0c\u512a\u5148\u9078\u64c7\u7121GUI\u7684\u7121\u982d\u9a45\u52d5\u5957\u4ef6\uff0c\u6e1b\u5c11\u5716\u5f62\u885d\u7a81\uff1a\n
     chmod +x NVIDIA-Linux-x86_64-525.89.02.run\r\n.\/NVIDIA-Linux-x86_64-525.89.02.run --no-opengl --silent<\/code><\/pre>\n<\/li>\n
     \u5bb9\u5668\u57f7\u884c\u6642\u4fee\u5fa9<\/strong>\u5728Kubernetes\u53e2\u96c6\u4e2d\uff0c\u66f4\u65b0nvidia-device-plugin<\/code>\u4ee5\u5339\u914d\u9a45\u52d5\u7248\u672c\u3002\u9a57\u8b49daemonset<\/code>\u914d\u7f6e\uff0c\u907f\u514d\u9999\u6e2f\u591a\u7bc0\u9ede\u4f48\u7f72\u4e2d\u7684GPU\u5206\u914d\u5931\u6557\u3002<\/li>\n<\/ol>\n
     2. \u74b0\u5883\u8207\u786c\u9ad4\u4fee\u5fa9<\/h3>\n
     \u61c9\u5c0d\u9999\u6e2f\u7684\u6c23\u5019\u6311\u6230\u662f\u9810\u9632\u6545\u969c\u5fa9\u767c\u7684\u95dc\u9375\uff1a<\/p>\n
     \n
     \u5373\u6642\u6563\u71b1\u63aa\u65bd<\/strong>\u5728\u4f3a\u670d\u5668\u6a5f\u67b6\u4e2d\u90e8\u7f72\u9ad8\u98a8\u91cf\u8ef8\u6d41\u98a8\u6247\u589e\u5f37\u6c23\u6d41\u3002\u5728\u4f3a\u670d\u5668\u8a17\u7ba1\u6a5f\u623f\u4e2d\uff0c\u9ad8\u6eab\u671f\u9593\u9700\u8207\u8cc7\u6599\u4e2d\u5fc3\u5de5\u4f5c\u4eba\u54e1\u5354\u8abf\uff0c\u8abf\u6574\u901a\u9053\u5bc6\u9589\u7cfb\u7d71\u3002<\/li>\n
     \u786c\u9ad4\u66f4\u63db\u6d41\u7a0b<\/strong>\u91dd\u5c0d\u4f3a\u670d\u5668\u79df\u7528\u74b0\u5883\u4e2d\u7684\u6545\u969cGPU\uff0c\u9075\u5faa\u4ee5\u4e0b\u6b65\u9a5f\uff1a\n
     \n
     \u62c6\u5378\u524d\u4f7f\u7528nvidia-smi -e 1<\/code>\u5099\u4efd\u97cc\u9ad4<\/li>\n
     \u78ba\u4fdd\u66ff\u63dbGPU\u8207\u539f\u578b\u865f\u4e00\u81f4\uff0c\u907f\u514dPCIe\u901a\u9053\u914d\u7f6e\u554f\u984c<\/li>\n
     \u70ba\u9999\u6e2f\u4f3a\u670d\u5668\u4f48\u7f72\u4e2d\u7684\u65b0\u786c\u9ad4\u91cd\u65b0\u914d\u7f6eBIOS\/UEFI\u8a2d\u5b9a<\/li>\n<\/ol>\n<\/li>\n
     \u9577\u671f\u6563\u71b1\u512a\u5316<\/strong>\u5c0d\u65bcAI\u5de5\u4f5c\u8ca0\u8f09\uff0c\u53ef\u8003\u616e\u70ba\u4f3a\u670d\u5668\u6539\u88dd\u6db2\u51b7\u7cfb\u7d71\u3002\u6d78\u6c92\u5f0f\u51b7\u537b\u5728\u9999\u6e2f\u9ad8\u6fd5\u5ea6\u74b0\u5883\u4e2d\uff0c\u53ef\u4f7fGPU\u6eab\u5ea6\u964d\u4f4e30-40\u2103\u3002<\/li>\n<\/ul>\n
     [\/vc_column_text][\/vc_column][\/vc_row][vc_row el_class=”blog-detail-section”][vc_column][vc_column_text css=””]<\/p>\n
     \u9999\u6e2f\u4f3a\u670d\u5668\u904b\u7dad\u4e2d\u7684GPU\u6545\u969c\u4e3b\u52d5\u9810\u9632\u7b56\u7565<\/strong><\/h2>\n
     1. \u5373\u6642\u76e3\u63a7\u67b6\u69cb<\/h3>\n
     \u4f48\u7f72\u53ef\u9760\u7684\u76e3\u63a7\u5806\u758a\u662f\u9810\u6e2c\u6027\u7dad\u8b77\u7684\u6838\u5fc3\uff1a<\/p>\n
     \n
     Prometheus\u914d\u7f6e<\/strong>\u4f7f\u7528\u4ee5\u4e0b\u7d00\u9304\u898f\u5247\u76e3\u63a7GPU\u7279\u5b9a\u6307\u6a19\uff1a\n
     record: gpu_temp_warning\r\nexpr: nvidia_gpu_temp_celsius > 80\r\nrecord: gpu_memory_alert\r\nexpr: nvidia_gpu_memory_used_bytes \/ nvidia_gpu_memory_total_bytes * 100 > 90<\/code><\/pre>\n<\/li>\n
     Grafana\u5100\u8868\u677f\u8a2d\u5b9a<\/strong>\u5efa\u7acb\u5305\u542b\u4ee5\u4e0b\u5167\u5bb9\u7684\u76e3\u63a7\u9762\u677f\uff1a\n
     \n
     24\u5c0f\u6642GPU\u6eab\u5ea6\u8da8\u52e2<\/li>\n
     \u4f3a\u670d\u5668\u53e2\u96c6\u4e2d\u9a45\u52d5\u7248\u672c\u4e00\u81f4\u6027<\/li>\n
     \u9999\u6e2f\u8cc7\u6599\u4e2d\u5fc3\u9ad8\u5cf0\u8ca0\u8f09\u671f\u9593\u7684GPU\u8a18\u61b6\u9ad4\u983b\u5bec\u4f7f\u7528\u7387<\/li>\n<\/ol>\n<\/li>\n
     \u544a\u8b66\u7b56\u7565<\/strong>\u914d\u7f6e\u591a\u7d1a\u544a\u8b66\u2014\u201475\u2103\u89f8\u767c\u8b66\u544a\uff0c85\u2103\u89f8\u767c\u7dca\u6025\u544a\u8b66\uff0c\u4ee5\u9069\u61c9\u9999\u6e2f\u74b0\u5883\u6eab\u5ea6\u7684\u6ce2\u52d5\u3002<\/li>\n<\/ul>\n
     2. \u7248\u672c\u63a7\u5236\u8207\u7dad\u8b77\u6d41\u7a0b<\/h3>\n
     \u7ba1\u7406\u9a45\u52d5\u8207\u7cfb\u7d71\u66f4\u65b0\uff0c\u6e1b\u5c11\u696d\u52d9\u4e2d\u65b7\uff1a<\/p>\n
     \n
     \u7dad\u8b77\u8996\u7a97<\/strong>\u5c07GPU\u9a45\u52d5\u66f4\u65b0\u5b89\u6392\u5728\u975e\u9ad8\u5cf0\u6642\u6bb5\uff0c\u907f\u514d\u5f71\u97ff\u9999\u6e2f\u7684\u570b\u969b\u696d\u52d9\u904b\u4f5c\u3002<\/li>\n
     \u7248\u672c\u76f8\u5bb9\u6027\u77e9\u9663<\/strong>\u7dad\u8b77\u8868\u683c\u7d00\u9304\u4ee5\u4e0b\u5c0d\u61c9\u95dc\u4fc2\uff1a\n
     \n
     \u6838\u5fc3\u7248\u672c\u8207\u76f8\u5bb9GPU\u9a45\u52d5<\/li>\n
     \u5bb9\u5668\u57f7\u884c\u6642\u7248\u672c\u8207nvidia-container-toolkit\u7248\u672c<\/li>\n
     \u97cc\u9ad4\u7248\u672c\u8207\u9999\u6e2f\u4f3a\u670d\u5668\u578b\u865f\u7684\u786c\u9ad4\u76f8\u5bb9\u6027<\/li>\n<\/ul>\n<\/li>\n
     \u81ea\u52d5\u5316\u6e2c\u8a66<\/strong>\u4f7f\u7528CI\/CD\u7ba1\u7dda\u9a57\u8b49\u66f4\u65b0\u5f8c\u7684GPU\u529f\u80fd\u3002\u57f7\u884cCUDA\u57fa\u6e96\u6e2c\u8a66\u548c3D\u6e32\u67d3\u6307\u4ee4\u7a3f\uff0c\u78ba\u4fdd\u6548\u80fd\u4e00\u81f4\u6027\u3002<\/li>\n<\/ul>\n
     3. \u5099\u63f4\u8207\u6545\u969c\u8f49\u79fb\u6a5f\u5236<\/h3>\n
     \u70ba\u9999\u6e2f\u4f3a\u670d\u5668\u67b6\u69cb\u5efa\u69cb\u5f48\u6027\uff1a<\/p>\n
     \n
     GPU\u6545\u969c\u8f49\u79fb\u6307\u4ee4\u7a3f<\/strong>\u5efa\u7acbsystemd\u55ae\u5143\u76e3\u63a7GPU\u5065\u5eb7\u72c0\u614b\uff1a\n
     [Unit]\r\nDescription=GPU Health Monitor\r\nAfter=multi-user.target\r\n\r\n[Service]\r\nType=simple\r\nExecStart=\/usr\/bin\/bash -c \"while true; do \r\n  if nvidia-smi | grep -q 'Failed'; then \r\n    systemctl restart gpu-failover.target; \r\n  fi; sleep 30; \r\ndone\"<\/code><\/pre>\n<\/li>\n
     \u5340\u57df\u5099\u63f4<\/strong>\u5c0d\u65bc\u6838\u5fc3\u696d\u52d9\u61c9\u7528\uff0c\u5728\u9999\u6e2f\u4e0d\u540c\u5340\u57df\u7684\u8cc7\u6599\u4e2d\u5fc3\u9593\u8907\u88fd\u5de5\u4f5c\u8ca0\u8f09\u3002\u4f7f\u7528BGP\u8def\u7531\u5be6\u73fe99.99% SLA\u8981\u6c42\u5167\u7684\u6545\u969c\u8f49\u79fb\u3002<\/li>\n
     \u5099\u7528\u786c\u9ad4\u5132\u5099<\/strong>\u5728\u4f3a\u670d\u5668\u8a17\u7ba1\u4f48\u7f72\u4e2d\uff0c\u6bcf\u500b\u6a5f\u67b6\u81f3\u5c11\u5132\u5099\u4e00\u584a\u5099\u7528GPU\uff0c\u4ee5\u7e2e\u77ed\u786c\u9ad4\u6545\u969c\u5f8c\u7684\u5e73\u5747\u4fee\u5fa9\u6642\u9593\uff08MTTR\uff09\u3002<\/li>\n<\/ul>\n
     [\/vc_column_text][\/vc_column][\/vc_row][vc_row el_class=”blog-detail-section”][vc_column][vc_column_text css=””]<\/p>\n
     \u7d50\u8a9e\uff1a\u9999\u6e2f\u4f3a\u670d\u5668\u751f\u614b\u4e2dGPU\u7ba1\u7406\u7684\u6700\u4f73\u5be6\u8e10<\/strong><\/h2>\n
     \u5728\u9999\u6e2f\u7368\u7279\u7684\u4f3a\u670d\u5668\u74b0\u5883\u4e2d\u7ba1\u7406GPU\u5065\u5eb7\uff0c\u9700\u8981\u6280\u8853\u5c08\u696d\u77e5\u8b58\u8207\u74b0\u5883\u610f\u8b58\u7684\u7d50\u5408\u3002\u6280\u8853\u4eba\u54e1\u7684\u6838\u5fc3\u8981\u9ede\u5305\u62ec\uff1a\u512a\u5148\u8655\u7406\u6563\u71b1\u7ba1\u7406\u3001\u56b4\u683c\u63a7\u5236\u9a45\u52d5\u7248\u672c\u3001\u4f48\u7f72\u9069\u914d\u672c\u5730\u6c23\u5019\u6311\u6230\u7684\u4e3b\u52d5\u76e3\u63a7\u3002\u5c07\u9019\u4e9b\u7b56\u7565\u878d\u5165\u65e5\u5e38\u904b\u7dad\uff0c\u5718\u968a\u53ef\u6700\u5927\u9650\u5ea6\u6e1b\u5c11GPU\u76f8\u95dc\u505c\u6a5f\u6642\u9593\uff0c\u78ba\u4fdd\u9999\u6e2f\u4f3a\u670d\u5668\u79df\u7528\u8207\u8a17\u7ba1\u670d\u52d9\u7684\u6700\u4f73\u6548\u80fd\u3002<\/p>\n
     \u5c0d\u65bc\u6301\u7e8c\u7684GPU\u7dad\u8b77\uff0c\u5efa\u8b70\u5efa\u7acb\u4ee5\u4e0b\u5e38\u898f\u6d41\u7a0b\uff1a<\/p>\n
     \n
     \u6bcf\u6708\u9032\u884c\u6eab\u5ea6\u6aa2\u67e5\u8207\u7070\u5875\u6e05\u7406<\/li>\n
     \u6bcf\u5b63\u5ea6\u5c0d\u7167NVIDIA\/AMD\u7684\u9999\u6e2f\u5730\u5340\u5c08\u9805\u5efa\u8b70\uff0c\u5be9\u67e5\u9a45\u52d5\u7248\u672c<\/li>\n
     \u6bcf\u5e74\u898f\u5283\u786c\u9ad4\u66f4\u65b0\uff0c\u61c9\u5c0d\u71b1\u5e36\u6c23\u5019\u4e0b\u7684\u8a2d\u5099\u8001\u5316\u554f\u984c<\/li>\n<\/ul>\n
     \u5c07GPU\u6545\u969c\u9810\u9632\u8996\u70ba\u7cfb\u7d71\u6027\u6311\u6230\u800c\u975e\u88ab\u52d5\u61c9\u5c0d\u4efb\u52d9\uff0c\u4f01\u696d\u624d\u80fd\u5728\u9999\u6e2f\u7af6\u722d\u6fc0\u70c8\u7684\u4f3a\u670d\u5668\u79df\u7528\u8207\u8a17\u7ba1\u5e02\u5834\u4e2d\uff0c\u7dad\u6301\u9ad8\u6a19\u6e96\u7684\u670d\u52d9\u53ef\u9760\u6027\u3002<\/p>\n
     [\/vc_column_text][\/vc_column][\/vc_row]<\/p>\n<\/div>","protected":false},"excerpt":{"rendered":"
     [vc_row el_class=”blog-detail-section”][vc_ […]<\/p>\n
     Read More…<\/a><\/p>\n","protected":false},"author":1,"featured_media":27558,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_acf_changed":false,"footnotes":""},"categories":[61],"tags":[8977,8978,8485,259],"class_list":["post-27563","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-hong-kong-dedicated-server-tc","tag-gpu-bugs-diagnosis-tc","tag-gpu-driver-fix-tc","tag-server-gpu-optimization-tc","tag-259"],"acf":[],"yoast_head":"\n