{"id":24604,"date":"2021-11-28T16:36:11","date_gmt":"2021-11-28T08:36:11","guid":{"rendered":"https:\/\/aqzt.com\/24604.html"},"modified":"2021-11-28T16:36:11","modified_gmt":"2021-11-28T08:36:11","slug":"hadoop%e6%80%a7%e8%83%bd%e8%b0%83%e4%bc%98","status":"publish","type":"post","link":"https:\/\/aqzt.com\/24604.html","title":{"rendered":"hadoop\u6027\u80fd\u8c03\u4f18"},"content":{"rendered":"<h1>\u76ee \u5f55<\/h1>\n<p>12 \u6027\u80fd\u8c03\u4f18<\/p>\n<\/p>\n<p>12.1 \u914d\u7f6e\u539f\u5219<\/p>\n<p>12.2 Manager<\/p>\n<p>12.2.1 \u63d0\u5347Manager\u914d\u7f6e\u670d\u52a1\u53c2\u6570\u7684\u6548\u7387<\/p>\n<p>12.2.2 \u6839\u636e\u96c6\u7fa4\u8282\u70b9\u6570\u4f18\u5316Manager\u914d\u7f6e<\/p>\n<p>12.3 HBase<\/p>\n<p>12.3.1 \u63d0\u5347BulkLoad\u6548\u7387<\/p>\n<p>12.3.2 \u63d0\u5347\u8fde\u7eedput\u573a\u666f\u6027\u80fd<\/p>\n<p>12.3.3 Put\u548cScan\u6027\u80fd\u7efc\u5408\u8c03\u4f18<\/p>\n<p>12.3.4 \u63d0\u5347\u5b9e\u65f6\u5199\u6570\u636e\u6548\u7387<\/p>\n<p>12.3.5 \u63d0\u5347\u5b9e\u65f6\u8bfb\u6570\u636e\u6548\u7387<\/p>\n<p>12.3.6 JVM\u53c2\u6570\u4f18\u5316<\/p>\n<p>12.4 HDFS<\/p>\n<p>12.4.1 \u63d0\u5347\u5199\u6027\u80fd<\/p>\n<p>12.4.2 JVM\u53c2\u6570\u4f18\u5316<\/p>\n<p>12.4.3 \u4f7f\u7528\u5ba2\u6237\u7aef\u5143\u6570\u636e\u7f13\u5b58\u63d0\u9ad8\u8bfb\u53d6\u6027\u80fd<\/p>\n<p>12.4.4 \u4f7f\u7528\u5f53\u524d\u6d3b\u52a8\u7f13\u5b58\u63d0\u5347\u5ba2\u6237\u7aef\u4e0eNameNode\u7684\u8fde\u63a5\u6027\u80fd<\/p>\n<p>12.5 Hive<\/p>\n<p>12.5.1 \u5efa\u7acb\u8868\u5206\u533a<\/p>\n<p>12.5.2 Join\u4f18\u5316<\/p>\n<p>12.5.3 Group By\u4f18\u5316<\/p>\n<p>12.5.4 \u6570\u636e\u5b58\u50a8\u4f18\u5316<\/p>\n<p>12.5.5 SQL\u4f18\u5316<\/p>\n<p>12.5.6 \u4f7f\u7528Hive CBO\u4f18\u5316\u67e5\u8be2<\/p>\n<p>12.6 Kafka<\/p>\n<p>12.6.1 Kafka\u6027\u80fd\u8c03\u4f18<\/p>\n<p>12.7 MapReduce<\/p>\n<p>12.7.1 \u591aCPU\u5185\u6838\u4e0b\u7684\u8c03\u4f18\u914d\u7f6e<\/p>\n<p>12.7.2 \u786e\u5b9aJob\u57fa\u7ebf<\/p>\n<p>12.7.3 Shuffle\u8c03\u4f18<\/p>\n<p>12.7.4 \u5927\u4efb\u52a1\u7684AM\u8c03\u4f18<\/p>\n<p>12.7.5 \u63a8\u6d4b\u6267\u884c<\/p>\n<p>12.7.6 \u901a\u8fc7\u201cSlow Start\u201d\u8c03\u4f18<\/p>\n<p>12.7.7 MR job commit\u9636\u6bb5\u4f18\u5316<\/p>\n<p>12.8 Solr<\/p>\n<p>12.8.1 \u7d22\u5f15\u96c6\u5206\u7247\u5212\u5206\u5efa\u8bae<\/p>\n<p>12.8.2 Solr\u516c\u5171\u8bfb\u5199\u8c03\u4f18\u5efa\u8bae<\/p>\n<p>12.8.3 Solr over HBase\u8c03\u4f18\u5efa\u8bae<\/p>\n<p>12.8.4 Solr over HDFS\u8c03\u4f18\u5efa\u8bae<\/p>\n<p>12.9 Spark<\/p>\n<p>12.9.1 Spark Core\u8c03\u4f18<\/p>\n<p>12.9.1.1 \u6570\u636e\u5e8f\u5217\u5316<\/p>\n<p>12.9.1.2 \u914d\u7f6e\u5185\u5b58<\/p>\n<p>12.9.1.3 \u8bbe\u7f6e\u5e76\u884c\u5ea6<\/p>\n<p>12.9.1.4 \u4f7f\u7528\u5e7f\u64ad\u53d8\u91cf<\/p>\n<p>12.9.1.5 \u4f7f\u7528External Shuffle Service\u63d0\u5347\u6027\u80fd<\/p>\n<p>12.9.1.6 Yarn\u6a21\u5f0f\u4e0b\u52a8\u6001\u8d44\u6e90\u8c03\u5ea6<\/p>\n<p>12.9.1.7 \u914d\u7f6e\u8fdb\u7a0b\u53c2\u6570<\/p>\n<p>12.9.1.8 \u8bbe\u8ba1DAG<\/p>\n<p>12.9.1.9 \u7ecf\u9a8c\u603b\u7ed3<\/p>\n<p>12.9.1.10 \u4f18\u5316\u8fd4\u56de\u67e5\u8be2\u7ed3\u679c\u8fd4\u56de\u5927\u91cf\u6570\u636e\u7684\u573a\u666f<\/p>\n<p>12.9.2 SQL\u548cDataFrame\u8c03\u4f18<\/p>\n<p>12.9.2.1 Spark SQL join\u4f18\u5316<\/p>\n<p>12.9.2.2 \u4f18\u5316\u6570\u636e\u503e\u659c\u573a\u666f\u4e0b\u7684Spark SQL\u6027\u80fd<\/p>\n<p>12.9.2.3 \u4f18\u5316\u5c0f\u6587\u4ef6\u573a\u666f\u4e0b\u7684Spark SQL\u6027\u80fd<\/p>\n<p>12.9.2.4 INSERT&#8230;SELECT\u64cd\u4f5c\u8c03\u4f18<\/p>\n<p>12.9.2.5 \u591a\u5e76\u53d1JDBC\u5ba2\u6237\u7aef\u8fde\u63a5JDBCServer<\/p>\n<p>12.9.2.6 Parquet\u5143\u6570\u636ebuildScan\u4f18\u5316<\/p>\n<p>12.9.2.7 ParquetRelation InputSplits\u4f18\u5316<\/p>\n<p>12.9.2.8 Limit\u4f18\u5316<\/p>\n<p>12.9.2.9 LimitScan\u4f18\u5316<\/p>\n<p>12.9.2.10 \u9884\u5148Broadcast\u5c0f\u8868\u4f18\u5316<\/p>\n<p>12.9.3 Spark Streaming\u8c03\u4f18<\/p>\n<p>12.9.4 Spark CBO\u8c03\u4f18<\/p>\n<p>12.9.5 Carbon\u6027\u80fd\u8c03\u4f18<\/p>\n<p>12.10 Storm<\/p>\n<p>12.10.1 Storm\u6027\u80fd\u8c03\u4f18<\/p>\n<p>12.11 YARN<\/p>\n<p>12.11.1 \u901a\u8fc7\u5bb9\u5668\u53ef\u91cd\u7528\u6027\u63d0\u9ad8\u4efb\u52a1\u7684\u5b8c\u6210\u6548\u7387<\/p>\n<p>12.11.2 \u62a2\u5360\u4efb\u52a1<\/p>\n<p>12.11.3 \u4efb\u52a1\u4f18\u5148\u7ea7<\/p>\n<p>12.11.4 \u8282\u70b9\u914d\u7f6e\u8c03\u4f18<\/p>\n<p>12.11.5 JVM\u53c2\u6570\u4f18\u5316<\/p>\n<h1>12\u00a0\u6027\u80fd\u8c03\u4f18<\/h1>\n<ul>\n<li><strong>12.1 \u914d\u7f6e\u539f\u5219<\/strong><\/li>\n<li><strong>12.2 Manager<\/strong><\/li>\n<li><strong>12.3 HBase<\/strong><\/li>\n<li><strong>12.4 HDFS<\/strong><\/li>\n<li><strong>12.5 Hive<\/strong><\/li>\n<li><strong>12.6 Kafka<\/strong><\/li>\n<li><strong>12.7 MapReduce<\/strong><\/li>\n<li><strong>12.8 Solr<\/strong><\/li>\n<li><strong>12.9 Spark<\/strong><\/li>\n<li><strong>12.10 Storm<\/strong><\/li>\n<li><strong>12.11 YARN<\/strong><\/li>\n<\/ul>\n<h2>12.1\u00a0\u914d\u7f6e\u539f\u5219<\/h2>\n<h3>\u5982\u4f55\u53d1\u6325\u96c6\u7fa4\u6700\u4f73\u6027\u80fd<\/h3>\n<p><strong>\u539f\u52191\uff1aCPU\u6838\u6570\u5206\u914d\u539f\u5219<\/strong><\/p>\n<ul>\n<li>\u6570\u636e\u8282\u70b9\uff1a\u5efa\u8bae\u9884\u75592\uff5e4\u4e2a\u6838\u7ed9OS\u548c\u5176\u4ed6\u8fdb\u7a0b(\u6570\u636e\u5e93\uff0cHBase\u7b49)\u5916\uff0c\u5176\u4ed6\u7684\u6838\u5206\u914d\u7ed9YARN\u3002<\/li>\n<li>\u63a7\u5236\u8282\u70b9\uff1a\u7531\u4e8e\u8fd0\u884c\u7684\u8fdb\u7a0b\u8f83\u591a\uff0c\u5efa\u8bae\u9884\u75596\uff5e8\u4e2a\u6838\u3002<\/li>\n<\/ul>\n<p><strong>\u539f\u52192\uff1a\u5185\u5b58\u5206\u914d<\/strong><\/p>\n<p>\u9664\u4e86\u5206\u914d\u7ed9OS\u3001\u5176\u4ed6\u670d\u52a1\u7684\u5185\u5b58\u5916\uff0c\u5269\u4f59\u7684\u8d44\u6e90\u5e94\u5c3d\u91cf\u5206\u914d\u7ed9YARN\u3002<\/p>\n<p><strong>\u539f\u52193\uff1a\u865a\u62dfCPU\u4e2a\u6570\u5206\u914d<\/strong><\/p>\n<p>\u8282\u70b9\u4e0aYARN\u53ef\u4f7f\u7528\u7684\u865a\u62dfCPU\u4e2a\u6570\u5efa\u8bae\u914d\u7f6e\u4e3a\u903b\u8f91\u6838\u6570\u76841.5\uff5e2\u500d\u4e4b\u95f4\u3002\u5982\u679c\u4e0a\u5c42\u8ba1\u7b97\u5e94\u7528\u5bf9CPU\u7684\u8ba1\u7b97\u80fd\u529b\u8981\u6c42\u4e0d\u9ad8\uff0c\u53ef\u4ee5\u914d\u7f6e\u4e3a2\u500d\u7684\u903b\u8f91CPU\u3002<\/p>\n<p><strong>\u539f\u52194\uff1a\u63d0\u9ad8\u78c1\u76d8IO\u541e\u5410\u7387<\/strong><\/p>\n<p>\u5c3d\u53ef\u80fd\u6302\u8f7d\u8f83\u591a\u7684\u76d8\uff0c\u4ee5\u63d0\u9ad8\u78c1\u76d8IO\u541e\u5410\u7387\u3002<\/p>\n<h3>\u5f71\u54cd\u6027\u80fd\u7684\u56e0\u7d20<\/h3>\n<p><strong>\u56e0\u7d201\uff1a\u6587\u4ef6\u670d\u52a1\u5668\u78c1\u76d8I\/O<\/strong><\/p>\n<p>\u4e00\u822c\u78c1\u76d8\u987a\u5e8f\u8bfb\u5199\u7684\u901f\u5ea6\u4e3a\u767e\u5146\u7ea7\u522b\uff0c\u5982\u7b2c\u4e8c\u4ee3SATA\u76d8\u987a\u5e8f\u8bfb\u7684\u7406\u8bba\u901f\u5ea6\u4e3a300Mbps\uff0c\u53ea\u4ece\u4e00\u4e2a\u76d8\u91cc\u8bfb\uff0c\u82e5\u60f3\u8fbe\u52301Gbps\u6bcf\u79d2\u7684\u5bfc\u5165\u901f\u5ea6\u662f\u4e0d\u53ef\u80fd\u7684\u3002\u5e76\u4e14\u82e5\u4ece\u4e00\u4e2a\u78c1\u76d8\u8bfb\uff0c\u5355\u7eaf\u4f9d\u9760\u589e\u52a0map\u6570\u6765\u63d0\u9ad8\u5bfc\u5165\u901f\u7387\u4e5f\u4e0d\u4e00\u5b9a\u53ef\u4ee5\u3002\u56e0\u4e3a\u968f\u7740map\u6570\u53d8\u591a\uff0c\u5bf9\u4e8e\u4e00\u4e2a\u78c1\u76d8\u91cc\u7684\u6587\u4ef6\u8bfb\uff0c\u76f8\u5f53\u7531\u987a\u5e8f\u8bfb\u53d8\u6210\u4e86\u968f\u673a\u8bfb\uff0cmap\u6570\u8d8a\u591a\uff0c\u78c1\u76d8\u8bfb\u53d6\u6587\u4ef6\u7684\u968f\u673a\u6027\u8d8a\u5f3a\uff0c\u8bfb\u53d6\u6027\u80fd\u53cd\u800c\u8d8a\u5dee\u3002\u5982\u968f\u673a\u8bfb\u6700\u5dee\u53ef\u53d8\u6210800Kbps\u3002 \u56e0\u6b64\u9700\u8981\u60f3\u529e\u6cd5\u589e\u5927\u6587\u4ef6\u670d\u52a1\u5668\u7684\u78c1\u76d8IO\u8bfb\u6548\u7387\uff0c\u53ef\u4ee5\u4f7f\u7528\u4e13\u4e1a\u7684\u6587\u4ef6\u670d\u52a1\u5668\uff0c\u5982NAS\u7cfb\u7edf\uff0c\u6216\u8005\u4f7f\u7528\u66f4\u7b80\u5355\u7684\u65b9\u6cd5\uff0c\u628a\u591a\u4e2a\u78c1\u76d8\u8fdb\u884cRaid0\u6216\u8005Raid5\u3002<\/p>\n<p><strong>\u56e0\u7d202\uff1a\u6587\u4ef6\u670d\u52a1\u5668\u7f51\u7edc\u5e26\u5bbd<\/strong><\/p>\n<p>\u5355\u4e2a\u6587\u4ef6\u670d\u52a1\u5668\u7684\u7f51\u7edc\u5e26\u5bbd\u8d8a\u5927\u8d8a\u597d\uff0c\u5efa\u8bae\u572810000Mb\/s\u4ee5\u4e0a\u3002<\/p>\n<p><strong>\u56e0\u7d203\uff1a\u96c6\u7fa4\u8282\u70b9\u786c\u4ef6\u914d\u7f6e<\/strong><\/p>\n<p>\u96c6\u7fa4\u8282\u70b9\u786c\u4ef6\u914d\u7f6e\u8d8a\u9ad8\uff0c\u5982CPU\u6838\u6570\u548c\u5185\u5b58\u90fd\u5f88\u591a\uff0c\u53ef\u4ee5\u589e\u5927\u540c\u65f6\u8fd0\u884c\u7684map\u6216reduce\u4e2a\u6570\uff0c\u5982\u679c\u5355\u4e2a\u8282\u70b9\u786c\u4ef6\u914d\u7f6e\u96be\u4ee5\u63d0\u5347\uff0c\u53ef\u4ee5\u589e\u52a0\u96c6\u7fa4\u8282\u70b9\u6570\u3002<\/p>\n<p><strong>\u56e0\u7d204\uff1aSFTP\u53c2\u6570\u914d\u7f6e<\/strong><\/p>\n<p>\u4e0d\u4f7f\u7528\u538b\u7f29\u3001\u52a0\u5bc6\u7b97\u6cd5\u4f18\u5148\u9009\u62e9aes128-cbc\uff0c\u5b8c\u6574\u6027\u6821\u9a8c\u7b97\u6cd5\u4f18\u5148\u9009\u62e9<a href=\"mailto:umac-64@openssh.com\">umac-64@openssh.com<\/a><\/p>\n<p><strong>\u56e0\u7d205\uff1a\u96c6\u7fa4\u53c2\u6570\u914d\u7f6e<\/strong><\/p>\n<p><strong>\u56e0\u7d206\uff1aLinux\u6587\u4ef6\u9884\u8bfb\u503c<\/strong><\/p>\n<p>\u8bbe\u7f6e\u78c1\u76d8\u6587\u4ef6\u9884\u8bfb\u503c\u5927\u5c0f\u4e3a16384\uff0c\u4f7f\u7528linux\u547d\u4ee4\uff1a<\/p>\n<p><strong>echo 16384 &gt; \/sys\/block\/sda\/queue\/read_ahead_kb<\/strong><\/p>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>sda\u8868\u793a\u5f53\u524d\u78c1\u76d8\u7684\u78c1\u76d8\u540d\u3002<\/p>\n<h2>12.2\u00a0Manager<\/h2>\n<p>\u00a0<\/p>\n<h3>12.2.1\u00a0\u63d0\u5347Manager\u914d\u7f6e\u670d\u52a1\u53c2\u6570\u7684\u6548\u7387<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5728\u5b89\u88c5\u96c6\u7fa4\u6216\u8005\u6269\u5bb9\u8282\u70b9\u4ee5\u540e\uff0c\u96c6\u7fa4\u4e2d\u53ef\u80fd\u6dfb\u52a0\u4e86\u8f83\u591a\u6570\u91cf\u7684\u8282\u70b9\u3002\u6b64\u65f6\u5982\u679c\u7cfb\u7edf\u7ba1\u7406\u5458\u5728FusionInsight Manager\u4e0a\u4fee\u6539\u670d\u52a1\u53c2\u6570\u3001\u4fdd\u5b58\u65b0\u914d\u7f6e\u5e76\u91cd\u542f\u670d\u52a1\u65f6\uff0cManager\u7684Controller\u8fdb\u7a0b\u53ef\u80fd\u5360\u7528\u5927\u91cf\u5185\u5b58\uff0c\u589e\u52a0\u4e86CPU\u5de5\u4f5c\u8d1f\u8377\uff0c\u7528\u6237\u9700\u8981\u7b49\u5f85\u4e00\u6bb5\u65f6\u95f4\u624d\u80fd\u5b8c\u6210\u53c2\u6570\u4fee\u6539\u3002\u7cfb\u7edf\u7ba1\u7406\u5458\u53ef\u4ee5\u6839\u636e\u5b9e\u9645\u4e1a\u52a1\u4f7f\u7528\u60c5\u51b5\uff0c\u624b\u52a8\u589e\u52a0Controller\u7684JVM\u542f\u52a8\u53c2\u6570\u4e2d\u5185\u5b58\u53c2\u6570\uff0c\u63d0\u5347\u914d\u7f6e\u670d\u52a1\u53c2\u6570\u7684\u6548\u7387\u3002<\/p>\n<p>\u5bf9\u7cfb\u7edf\u7684\u5f71\u54cd<\/p>\n<p>\u8be5\u64cd\u4f5c\u9700\u8981\u5728\u4e3b\u7ba1\u7406\u8282\u70b9\u91cd\u65b0\u542f\u52a8Controller\uff0c\u91cd\u542f\u671f\u95f4\u4f1a\u9020\u6210FusionInsight Manager\u6682\u65f6\u4e2d\u65ad\u3002\u5907\u7ba1\u7406\u8282\u70b9Controller\u65e0\u9700\u91cd\u542f\u3002<\/p>\n<p>\u524d\u63d0\u6761\u4ef6<\/p>\n<p>\u5df2\u786e\u8ba4\u4e3b\u5907\u7ba1\u7406\u8282\u70b9IP\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ol>\n<li>\u4f7f\u7528PuTTY\uff0c\u4ee5<strong>omm<\/strong>\u7528\u6237\u767b\u5f55\u4e3b\u7ba1\u7406\u8282\u70b9\u3002<\/li>\n<li>\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5207\u6362\u76ee\u5f55\u3002<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p><strong>cd ${BIGDATA_HOME}\/om-server\/om\/sbin<\/strong><\/p>\n<p>\u00a0<\/p>\n<ol>\n<li>\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u4fee\u6539Controller\u542f\u52a8\u53c2\u6570\u6587\u4ef6\u201ccontroller.sh\u201d\uff0c\u5e76\u4fdd\u5b58\u9000\u51fa\u3002<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p><strong>vi controller.sh<\/strong><\/p>\n<p>\u4fee\u6539\u914d\u7f6e\u9879\u201cJAVA_HEAP_MAX\u201d\u7684\u53c2\u6570\u503c\u3002\u4f8b\u5982\uff0c\u96c6\u7fa4\u4e2d\u5305\u542b\u4e86400\u4e2a\u4ee5\u4e0a\u7684\u8282\u70b9\uff0c\u5efa\u8bae\u4fee\u6539\u5982\u4e0b\uff0c\u8868\u793aController\u6700\u5927\u53ef\u4f7f\u75288GB\u5185\u5b58\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2JAVA_HEAP_MAX=-Xmx8192m<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u00a0<\/p>\n<ol>\n<li>\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u91cd\u65b0\u542f\u52a8Controller\u3002<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p><strong>sh ${BIGDATA_HOME}\/om-server\/om\/sbin\/restart-controller.sh<\/strong><\/p>\n<p>\u63d0\u793a\u4ee5\u4e0b\u4fe1\u606f\u8868\u793a\u547d\u4ee4\u6267\u884c\u6210\u529f\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2End into start-controller.sh<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u6267\u884c<strong>sh ${BIGDATA_HOME}\/om-server\/om\/sbin\/status-oms.sh<\/strong>\uff0c\u67e5\u770bController\u7684\u201cResHAStatus\u201d\u662f\u5426\u4e3a\u201cNormal\u201d\uff0c\u5e76\u53ef\u4ee5\u91cd\u65b0\u767b\u5f55FusionInsight Manager\u8868\u793a\u91cd\u542f\u6210\u529f\u3002<\/p>\n<p>\u00a0<\/p>\n<ol>\n<li>\u4f7f\u7528PuTTY\uff0c\u4ee5<strong>omm<\/strong>\u7528\u6237\u767b\u5f55\u5907\u7ba1\u7406\u8282\u70b9\uff0c\u5e76\u91cd\u590d\u6b65\u9aa4 2\uff5e\u6b65\u9aa4 3\u3002<\/li>\n<\/ol>\n<h3>12.2.2\u00a0\u6839\u636e\u96c6\u7fa4\u8282\u70b9\u6570\u4f18\u5316Manager\u914d\u7f6e<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>FusionInsight\u96c6\u7fa4\u89c4\u6a21\u4e0d\u540c\u65f6\uff0cManager\u76f8\u5173\u53c2\u6570\u5dee\u5f02\u8f83\u5927\u3002\u5728\u96c6\u7fa4\u5bb9\u91cf\u8c03\u6574\u524d\u6216\u8005\u5b89\u88c5\u96c6\u7fa4\u65f6\uff0c\u7528\u6237\u53ef\u4ee5\u624b\u52a8\u6307\u5b9aManager\u96c6\u7fa4\u8282\u70b9\u6570\uff0c\u7cfb\u7edf\u5c06\u81ea\u52a8\u8c03\u6574\u76f8\u5173\u8fdb\u7a0b\u53c2\u6570\u3002<\/p>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u5728\u5b89\u88c5\u96c6\u7fa4\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7Manager\u5b89\u88c5\u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u201ccluster_nodes_scale\u201d\u53c2\u6570\u6307\u5b9a\u96c6\u7fa4\u8282\u70b9\u6570\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ol>\n<li>\u4f7f\u7528PuTTY\uff0c\u4ee5<strong>omm<\/strong>\u7528\u6237\u767b\u5f55\u4e3b\u7ba1\u7406\u8282\u70b9\u3002<\/li>\n<li>\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5207\u6362\u76ee\u5f55\u3002<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p><strong>cd ${BIGDATA_HOME}\/om-server\/om\/sbin<\/strong><\/p>\n<p>\u00a0<\/p>\n<ol>\n<li>\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u67e5\u770b\u5f53\u524d\u96c6\u7fa4Manager\u76f8\u5173\u914d\u7f6e\u3002<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p><strong>sh oms_config_info.sh -q<\/strong><\/p>\n<p>\u00a0<\/p>\n<ol>\n<li>\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u6307\u5b9a\u5f53\u524d\u96c6\u7fa4\u7684\u8282\u70b9\u6570\u3002<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p>\u547d\u4ee4\u683c\u5f0f\uff1a**sh oms_config_info.sh -s\u00a0**<em>\u8282\u70b9\u6570<\/em><\/p>\n<p>\u4f8b\u5982\uff1a<\/p>\n<p><strong>sh oms_config_info.sh -s 10<\/strong><\/p>\n<p>\u6839\u636e\u754c\u9762\u63d0\u793a\uff0c\u8f93\u5165\u201cy\u201d\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/>6<br \/>7<br \/>8<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2The following configurations will be modified:<br \/>\n3 &nbsp; &nbsp; Module &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Parameter &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;Current &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; Target &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<br \/>\n4 &nbsp; &nbsp; Controller &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;controller.Xmx &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; 4096m &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;=&amp;gt; &nbsp; 8192m &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<br \/>\n5 &nbsp; &nbsp; Controller &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;controller.Xms &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; 1024m &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;=&amp;gt; &nbsp; 2048m &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <br \/>\n6 &nbsp; &nbsp; ...<br \/>\n7Do you really want to do this operation? (y\/n):<br \/>\n8<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u754c\u9762\u63d0\u793a\u4ee5\u4e0b\u4fe1\u606f\u8868\u793a\u914d\u7f6e\u66f4\u65b0\u6210\u529f\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2...<br \/>\n3Operation has been completed. Now restarting OMS server. &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;[done]<br \/>\n4Restarted oms server successfully.<br \/>\n5<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/><\/div><\/td><td><div class=\"text codecolorer\">1 \u00a0\u8bf4\u660e\uff1a <br \/>\n2<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<ul>\n<li>\u914d\u7f6e\u66f4\u65b0\u8fc7\u7a0b\u4e2d\uff0cOMS\u4f1a\u81ea\u52a8\u91cd\u542f\u3002<!-- -->\n<ul>\n<li>\u76f8\u8fd1\u6570\u91cf\u7684\u8282\u70b9\u89c4\u6a21\u5bf9\u5e94\u7684Manager\u76f8\u5173\u914d\u7f6e\u662f\u901a\u7528\u7684\uff0c\u4f8b\u5982100\u8282\u70b9\u53d8\u4e3a101\u8282\u70b9\uff0c\u5e76\u6ca1\u6709\u65b0\u7684\u914d\u7f6e\u9879\u9700\u8981\u5237\u65b0\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u00a0<\/p>\n<h2>12.3\u00a0HBase<\/h2>\n<h3>12.3.1\u00a0\u63d0\u5347BulkLoad\u6548\u7387<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u6279\u91cf\u52a0\u8f7d\u529f\u80fd\u91c7\u7528\u4e86MapReduce jobs\u76f4\u63a5\u751f\u6210\u7b26\u5408HBase\u5185\u90e8\u6570\u636e\u683c\u5f0f\u7684\u6587\u4ef6\uff0c\u7136\u540e\u628a\u751f\u6210\u7684StoreFiles\u6587\u4ef6\u52a0\u8f7d\u5230\u6b63\u5728\u8fd0\u884c\u7684\u96c6\u7fa4\u3002\u4f7f\u7528\u6279\u91cf\u52a0\u8f7d\u76f8\u6bd4\u76f4\u63a5\u4f7f\u7528HBase\u7684API\u4f1a\u8282\u7ea6\u66f4\u591a\u7684CPU\u548c\u7f51\u7edc\u8d44\u6e90\u3002<\/p>\n<p>ImportTSV\u662f\u4e00\u4e2aHBase\u7684\u8868\u6570\u636e\u52a0\u8f7d\u5de5\u5177\u3002<\/p>\n<p>\u524d\u63d0\u6761\u4ef6<\/p>\n<p>\u5728\u6267\u884c\u6279\u91cf\u52a0\u8f7d\u65f6\u9700\u8981\u901a\u8fc7\u201cDimporttsv.bulk.output\u201d\u53c2\u6570\u6307\u5b9a\u6587\u4ef6\u7684\u8f93\u51fa\u8def\u5f84\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a\u6267\u884c\u6279\u91cf\u52a0\u8f7d\u4efb\u52a1\u65f6\uff0c\u5728BulkLoad\u547d\u4ee4\u884c\u4e2d\u52a0\u5165\u5982\u4e0b\u53c2\u6570\u3002<\/p>\n<p>\u886812-1\u00a0\u589e\u5f3aBulkLoad\u6548\u7387\u7684\u914d\u7f6e\u9879<\/p>\n<p>-Dimporttsv.mapper.class<br \/>\n\u7528\u6237\u81ea\u5b9a\u4e49mapper\u901a\u8fc7\u628a\u952e\u503c\u5bf9\u7684\u6784\u9020\u4ecemapper\u79fb\u52a8\u5230reducer\u4ee5\u5e2e\u52a9\u63d0\u9ad8\u6027\u80fd\u3002mapper\u53ea\u9700\u8981\u628a\u6bcf\u4e00\u884c\u7684\u539f\u59cb\u6587\u672c\u53d1\u9001\u7ed9reducer\uff0creducer\u89e3\u6790\u6bcf\u4e00\u884c\u7684\u6bcf\u4e00\u6761\u8bb0\u5f55\u5e76\u521b\u5efa\u952e\u503c\u5bf9\u3002 \u8bf4\u660e\uff1a \u5f53\u8be5\u503c\u914d\u7f6e\u4e3a\u201corg.apache.hadoop.hbase.mapreduce.TsvImporterByteMapper\u201d\u65f6\uff0c\u53ea\u5728\u6267\u884c\u6ca1\u6709HBASE_CELL_VISIBILITY OR HBASE_CELL_TTL\u9009\u9879\u7684\u6279\u91cf\u52a0\u8f7d\u547d\u4ee4\u65f6\u4f7f\u7528\u3002\u4f7f\u7528\u201corg.apache.hadoop.hbase.mapreduce.TsvImporterByteMapper\u201d\u65f6\u53ef\u4ee5\u5f97\u5230\u66f4\u597d\u7684\u6027\u80fd\u3002<br \/>\norg.apache.hadoop.hbase.mapreduce.TsvImporterByteMapper \u548c org.apache.hadoop.hbase.mapreduce.TsvImporterTextMapper<\/p>\n<h3>12.3.2\u00a0\u63d0\u5347\u8fde\u7eedput\u573a\u666f\u6027\u80fd<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5bf9\u5927\u6279\u91cf\u3001\u8fde\u7eedput\u7684\u573a\u666f\uff0c\u914d\u7f6e\u4e0b\u9762\u7684\u4e24\u4e2a\u53c2\u6570\u4e3a\u201cfalse\u201d\u65f6\u80fd\u5927\u91cf\u63d0\u5347\u6027\u80fd\u3002<\/p>\n<ul>\n<li>\n<p>\u201chbase.regionserver.wal.durable.sync\u201d<\/p>\n<\/li>\n<li>\n<p>\u201chbase.regionserver.hfile.durable.sync\u201d<\/p>\n<\/li>\n<\/ul>\n<p>\u5f53\u63d0\u5347\u6027\u80fd\u65f6\uff0c\u7f3a\u70b9\u662f\u5bf9\u4e8eDataNode\uff08\u9ed8\u8ba4\u662f3\u4e2a\uff09\u540c\u65f6\u6545\u969c\u65f6\uff0c\u5b58\u5728\u5c0f\u6982\u7387\u6570\u636e\u4e22\u5931\u7684\u73b0\u8c61\u3002\u5bf9\u6570\u636e\u53ef\u9760\u6027\u8981\u6c42\u9ad8\u7684\u573a\u666f\u8bf7\u614e\u91cd\u914d\u7f6e\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; HBase &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<\/p>\n<p>\u886812-2\u00a0\u63d0\u5347\u8fde\u7eedput\u573a\u666f\u6027\u80fd\u7684\u53c2\u6570<\/p>\n<p>hbase.regionserver.wal.durable.sync<br \/>\n\u6bcf\u4e00\u6761wal\u662f\u5426\u6301\u4e45\u5316\u5230\u786c\u76d8\u3002<br \/>\ntrue<br \/>\nhbase.regionserver.hfile.durable.sync<br \/>\nhfile\u5199\u662f\u5426\u7acb\u5373\u6301\u4e45\u5316\u5230\u786c\u76d8\u3002<br \/>\ntrue<\/p>\n<h3>12.3.3\u00a0Put\u548cScan\u6027\u80fd\u7efc\u5408\u8c03\u4f18<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>HBase\u6709\u5f88\u591a\u4e0e\u8bfb\u5199\u6027\u80fd\u76f8\u5173\u7684\u914d\u7f6e\u53c2\u6570\u3002\u8bfb\u5199\u8bf7\u6c42\u8d1f\u8f7d\u4e0d\u540c\u7684\u60c5\u51b5\u4e0b\uff0c\u914d\u7f6e\u53c2\u6570\u9700\u8981\u8fdb\u884c\u76f8\u5e94\u7684\u8c03\u6574\uff0c\u672c\u7ae0\u8282\u65e8\u5728\u6307\u5bfc\u7528\u6237\u901a\u8fc7\u4fee\u6539RegionServer\u914d\u7f6e\u53c2\u6570\u8fdb\u884c\u8bfb\u5199\u6027\u80fd\u8c03\u4f18\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ul>\n<li>JVM GC\u53c2\u6570<\/li>\n<\/ul>\n<p>RegionServer GC_OPTS\u53c2\u6570\u8bbe\u7f6e\u5efa\u8bae\uff1a<\/p>\n<ul>\n<li>-Xms\u4e0e-Xmx\u8bbe\u7f6e\u76f8\u540c\u7684\u503c\uff0c\u9700\u8981\u6839\u636e\u5b9e\u9645\u60c5\u51b5\u8bbe\u7f6e\uff0c\u589e\u5927\u5185\u5b58\u53ef\u4ee5\u63d0\u9ad8\u8bfb\u5199\u6027\u80fd\uff0c\u53ef\u4ee5\u53c2\u8003\u53c2\u6570\u201chfile.block.cache.size\u201d\uff08\u89c1\u886812-4\uff09\u548c\u53c2\u6570\u201chbase.regionserver.global.memstore.size\u201d\uff08\u89c1\u886812-3\uff09\u7684\u4ecb\u7ecd\u8fdb\u884c\u8bbe\u7f6e\u3002<!-- -->\n<ul>\n<li>-XX:NewSize\u4e0e-XX:MaxNewSize\u8bbe\u7f6e\u76f8\u540c\u503c\uff0c\u5efa\u8bae\u4f4e\u8d1f\u8f7d\u573a\u666f\u4e0b\u8bbe\u7f6e\u4e3a\u201c512M\u201d\uff0c\u9ad8\u8d1f\u8f7d\u573a\u666f\u4e0b\u8bbe\u7f6e\u4e3a\u201c2048M\u201d\u3002<\/li>\n<li>-XX:CMSInitiatingOccupancyFraction\u5efa\u8bae\u8bbe\u7f6e\u4e3a\u201c100 * (hfile.block.cache.size + hbase.regionserver.global.memstore.size + 0.05)\u201d\uff0c\u6700\u5927\u503c\u4e0d\u8d85\u8fc790\u3002<\/li>\n<li>-XX:MaxDirectMemorySize\u8868\u793aJVM\u4f7f\u7528\u7684\u5806\u5916\u5185\u5b58\uff0c\u5efa\u8bae\u4f4e\u8d1f\u8f7d\u60c5\u51b5\u4e0b\u8bbe\u7f6e\u4e3a\u201c512M\u201d\uff0c\u9ad8\u8d1f\u8f7d\u60c5\u51b5\u4e0b\u8bbe\u7f6e\u4e3a\u201c2048M\u201d\u3002<\/li>\n<\/ul>\n<\/li>\n<li>Put\u76f8\u5173\u53c2\u6570<\/li>\n<\/ul>\n<p>RegionServer\u5904\u7406put\u8bf7\u6c42\u7684\u6570\u636e\uff0c\u4f1a\u5c06\u6570\u636e\u5199\u5165memstore\u548chlog\uff0c<\/p>\n<ul>\n<li>\u5f53memstore\u5927\u5c0f\u8fbe\u5230\u8bbe\u7f6e\u7684\u201chbase.hregion.memstore.flush.size\u201d\u53c2\u6570\u503c\u5927\u5c0f\u65f6\uff0cmemstore\u5c31\u4f1a\u5237\u65b0\u5230HDFS\u751f\u6210HFile\u3002<!-- -->\n<ul>\n<li>\u5f53\u5f53\u524dregion\u7684\u5217\u7c07\u7684HFile\u6570\u91cf\u8fbe\u5230\u201chbase.hstore.compaction.min\u201d\u53c2\u6570\u503c\u65f6\u4f1a\u89e6\u53d1compaction\u3002<\/li>\n<li>\u5f53\u5f53\u524dregion\u7684\u5217\u7c07HFile\u6570\u8fbe\u5230\u201chbase.hstore.blockingStoreFiles\u201d\u53c2\u6570\u503c\u65f6\u4f1a\u963b\u585ememstore\u5237\u65b0\u751f\u6210HFile\u7684\u64cd\u4f5c\uff0c\u5bfc\u81f4put\u8bf7\u6c42\u963b\u585e\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u886812-3\u00a0Put\u76f8\u5173\u53c2\u6570<\/p>\n<p>hbase.regionserver.wal.durable.sync<br \/>\n\u6bcf\u4e00\u6761wal\u662f\u5426\u6301\u4e45\u5316\u5230\u786c\u76d8\u3002 \u53c2\u8003\u63d0\u5347\u8fde\u7eedput\u573a\u666f\u6027\u80fd\u3002<br \/>\ntrue<br \/>\nhbase.regionserver.hfile.durable.sync<br \/>\nhfile\u5199\u662f\u5426\u7acb\u5373\u6301\u4e45\u5316\u5230\u786c\u76d8\u3002 \u53c2\u8003\u63d0\u5347\u8fde\u7eedput\u573a\u666f\u6027\u80fd\u3002<br \/>\ntrue<br \/>\nhbase.hregion.memstore.flush.size<br \/>\n\u5efa\u8bae\u8bbe\u7f6e\u4e3aHDFS\u5757\u5927\u5c0f\u7684\u6574\u6570\u500d\uff0c\u5728\u5185\u5b58\u8db3\u591fput\u8d1f\u8f7d\u5927\u60c5\u51b5\u4e0b\u53ef\u4ee5\u8c03\u6574\u589e\u5927\u3002\u5355\u4f4d\uff1a\u5b57\u8282\u3002<br \/>\n134217728<br \/>\nhbase.regionserver.global.memstore.size<br \/>\n\u5efa\u8bae\u8bbe\u7f6e\u4e3a\u201chbase.hregion.memstore.flush.size * \u5199\u6d3b\u8dc3region\u6570 \/ RegionServer GC -Xmx\u201d\u3002\u9ed8\u8ba4\u503c\u4e3a\u201c0.4\u201d\uff0c\u8868\u793a\u4f7f\u7528RegionServer GC -Xmx\u768440%\u3002<br \/>\n0.4<br \/>\nhbase.hstore.flusher.count<br \/>\nmemstore\u7684flush\u7ebf\u7a0b\u6570\uff0c\u5728put\u9ad8\u8d1f\u8f7d\u573a\u666f\u4e0b\u53ef\u4ee5\u9002\u5f53\u8c03\u5927\u3002<br \/>\n2<br \/>\nhbase.regionserver.thread.compaction.small<br \/>\nHFile compaction\u7ebf\u7a0b\u6570\uff0c\u5728put\u9ad8\u8d1f\u8f7d\u60c5\u51b5\u4e0b\u53ef\u4ee5\u9002\u5f53\u8c03\u5927\u3002<br \/>\n10<br \/>\nhbase.hstore.blockingStoreFiles<br \/>\n\u5f53\u5217\u7c07\u7684HFile\u6570\u8fbe\u5230\u8be5\u9608\u503c\uff0c\u963b\u585e\u8be5region\u7684\u6240\u6709\u64cd\u4f5c\uff0c\u76f4\u5230compcation\u5b8c\u6210\uff0c\u5728put\u9ad8\u8d1f\u8f7d\u573a\u666f\u4e0b\u53ef\u4ee5\u9002\u5f53\u8c03\u5927\u3002<br \/>\n15<\/p>\n<ul>\n<li>Scan\u76f8\u5173\u53c2\u6570\u886812-4\u00a0Scan\u76f8\u5173\u53c2\u6570<\/li>\n<\/ul>\n<p>hbase.client.scanner.timeout.period<br \/>\n\u5ba2\u6237\u7aef\u548cRegionServer\u7aef\u53c2\u6570\uff0c\u8868\u793ascan\u79df\u7ea6\u7684\u65f6\u95f4\uff0c\u5efa\u8bae\u8bbe\u7f6e\u4e3a60000ms\u7684\u6574\u6570\u500d\uff0c\u5728\u8bfb\u9ad8\u8d1f\u8f7d\u60c5\u51b5\u4e0b\u53ef\u4ee5\u9002\u5f53\u8c03\u5927\u3002\u5355\u4f4d\uff1a\u6beb\u79d2\u3002<br \/>\n60000<br \/>\nhfile.block.cache.size<br \/>\n\u6570\u636e\u7f13\u5b58\u6240\u5360\u7684RegionServer GC -Xmx\u767e\u5206\u6bd4\uff0c\u5728\u8bfb\u9ad8\u8d1f\u8f7d\u60c5\u51b5\u4e0b\u53ef\u4ee5\u9002\u5f53\u8c03\u5927\u4ee5\u589e\u5927\u7f13\u5b58\u547d\u4e2d\u7387\u4ee5\u63d0\u9ad8\u6027\u80fd\u3002\u9ed8\u8ba4\u503c\u4e3a\u201c0.25\u201d\uff0c\u8868\u793a\u4f7f\u7528RegionServer GC -Xmx\u768425%\u3002<br \/>\n0.25<\/p>\n<ul>\n<li>Handler\u76f8\u5173\u53c2\u6570\u886812-5\u00a0Handler\u76f8\u5173\u53c2\u6570<\/li>\n<\/ul>\n<p>hbase.regionserver.handler.count<br \/>\nRegionServer\u4e0a\u7684RPC\u670d\u52a1\u5668\u5b9e\u4f8b\u6570\uff0c\u5efa\u8bae\u8bbe\u7f6e\u4e3a200 ~ 400\u4e4b\u95f4\u3002<br \/>\n200<br \/>\nhbase.regionserver.metahandler.count<br \/>\nRegionServer\u4e2d\u5904\u7406\u4f18\u5148\u8bf7\u6c42\u7684\u7a0b\u5e8f\u5b9e\u4f8b\u7684\u6570\u91cf\uff0c\u5efa\u8bae\u8bbe\u7f6e\u4e3a200 ~ 400\u4e4b\u95f4\u3002<br \/>\n100<\/p>\n<h3>12.3.4\u00a0\u63d0\u5347\u5b9e\u65f6\u5199\u6570\u636e\u6548\u7387<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u9700\u8981\u628a\u6570\u636e\u5b9e\u65f6\u5199\u5165\u5230HBase\u4e2d\u6216\u8005\u5bf9\u4e8e\u5927\u6279\u91cf\u3001\u8fde\u7eedput\u7684\u573a\u666f\u3002<\/p>\n<p>\u524d\u63d0\u6761\u4ef6<\/p>\n<p>\u8c03\u7528HBase\u7684put\u6216delete\u63a5\u53e3\uff0c\u628a\u6570\u636e\u4fdd\u5b58\u5230HBase\u4e2d\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ul>\n<li><strong>\u5199\u6570\u636e\u670d\u52a1\u7aef\u8c03\u4f18<\/strong><\/li>\n<\/ul>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; HBase &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<br \/>\n\u886812-6\u00a0\u5f71\u54cd\u5b9e\u65f6\u5199\u6570\u636e\u914d\u7f6e\u9879<\/p>\n<p>hbase.regionserver.wal.durable.sync<br \/>\n\u63a7\u5236HLog\u6587\u4ef6\u5728\u5199\u5165\u5230HDFS\u65f6\u7684\u540c\u6b65\u7a0b\u5ea6\u3002\u5982\u679c\u4e3atrue\uff0cHDFS\u5728\u628a\u6570\u636e\u5199\u5165\u5230\u786c\u76d8\u540e\u624d\u8fd4\u56de\uff1b\u5982\u679c\u4e3afalse\uff0cHDFS\u5728\u628a\u6570\u636e\u5199\u5165OS\u7684\u7f13\u5b58\u540e\u5c31\u8fd4\u56de\u3002 \u628a\u8be5\u503c\u8bbe\u7f6e\u4e3afalse\u6bd4true\u5728\u5199\u5165\u6027\u80fd\u4e0a\u4f1a\u66f4\u4f18\u3002<br \/>\ntrue<br \/>\nhbase.regionserver.hfile.durable.sync<br \/>\n\u63a7\u5236HFile\u6587\u4ef6\u5728\u5199\u5165\u5230HDFS\u65f6\u7684\u540c\u6b65\u7a0b\u5ea6\u3002\u5982\u679c\u4e3atrue\uff0cHDFS\u5728\u628a\u6570\u636e\u5199\u5165\u5230\u786c\u76d8\u540e\u624d\u8fd4\u56de\uff1b\u5982\u679c\u4e3afalse\uff0cHDFS\u5728\u628a\u6570\u636e\u5199\u5165OS\u7684\u7f13\u5b58\u540e\u5c31\u8fd4\u56de\u3002 \u628a\u8be5\u503c\u8bbe\u7f6e\u4e3afalse\u6bd4true\u5728\u5199\u5165\u6027\u80fd\u4e0a\u4f1a\u66f4\u4f18\u3002<br \/>\ntrue<br \/>\nGC_OPTS<br \/>\nHBase\u5229\u7528\u5185\u5b58\u5b8c\u6210\u8bfb\u5199\u64cd\u4f5c\u3002\u63d0\u9ad8HBase\u5185\u5b58\u53ef\u4ee5\u6709\u6548\u63d0\u9ad8HBase\u6027\u80fd\u3002GC_OPTS\u4e3b\u8981\u9700\u8981\u8c03\u6574HeapSize\u7684\u5927\u5c0f\u548cNewSize\u7684\u5927\u5c0f\u3002\u8c03\u6574HeapSize\u5927\u5c0f\u7684\u65f6\u5019\uff0c\u5efa\u8bae\u5c06Xms\u548cXmx\u8bbe\u7f6e\u6210\u76f8\u540c\u7684\u503c\uff0c\u8fd9\u6837\u53ef\u4ee5\u907f\u514dJVM\u52a8\u6001\u8c03\u6574HeapSize\u5927\u5c0f\u7684\u65f6\u5019\u5f71\u54cd\u6027\u80fd\u3002\u8c03\u6574NewSize\u5927\u5c0f\u7684\u65f6\u5019\uff0c\u5efa\u8bae\u628a\u5176\u8bbe\u7f6e\u4e3aHeapSize\u5927\u5c0f\u76841\/9\u3002 HMaster\uff1a\u5f53HBase\u96c6\u7fa4\u89c4\u6a21\u8d8a\u5927\u3001Region\u6570\u91cf\u8d8a\u591a\u65f6\uff0c\u53ef\u4ee5\u9002\u5f53\u8c03\u5927HMaster\u7684GC_OPTS\u53c2\u6570\u3002 RegionServer\uff1aRegionServer\u9700\u8981\u7684\u5185\u5b58\u4e00\u822c\u6bd4HMaster\u8981\u5927\u3002\u5728\u5185\u5b58\u5145\u8db3\u7684\u60c5\u51b5\u4e0b\uff0cHeapSize\u53ef\u4ee5\u76f8\u5bf9\u8bbe\u7f6e\u5927\u4e00\u4e9b\u3002 \u8bf4\u660e\uff1a \u4e3bHMaster\u7684HeapSize\u4e3a4G\u7684\u65f6\u5019\uff0cHBase\u96c6\u7fa4\u53ef\u4ee5\u652f\u6301100000Region\u6570\u7684\u89c4\u6a21\u3002\u6839\u636e\u7ecf\u9a8c\u503c\uff0c\u5355\u4e2aRegionServer\u7684HeapSize\u4e0d\u5efa\u8bae\u8d85\u8fc720GB\u3002<br \/>\nHMaster\uff1a -Xms2G -Xmx2G -XX:NewSize=256M -XX:MaxNewSize=256M RegionServer\uff1a -Xms4G -Xmx4G -XX:NewSize=512M -XX:MaxNewSize=512M<br \/>\nhbase.regionserver.handler.count<br \/>\n\u8868\u793aRegionServer\u5728\u540c\u4e00\u65f6\u523b\u80fd\u591f\u5e76\u53d1\u5904\u7406\u591a\u5c11\u8bf7\u6c42\u3002\u5982\u679c\u8bbe\u7f6e\u8fc7\u9ad8\u4f1a\u5bfc\u81f4\u6fc0\u70c8\u7ebf\u7a0b\u7ade\u4e89\uff0c\u5982\u679c\u8bbe\u7f6e\u8fc7\u5c0f\uff0c\u8bf7\u6c42\u5c06\u4f1a\u5728RegionServer\u957f\u65f6\u95f4\u7b49\u5f85\uff0c\u964d\u4f4e\u5904\u7406\u80fd\u529b\u3002\u6839\u636e\u8d44\u6e90\u60c5\u51b5\uff0c\u9002\u5f53\u589e\u52a0\u5904\u7406\u7ebf\u7a0b\u6570\u3002 \u5efa\u8bae\u6839\u636eCPU\u7684\u4f7f\u7528\u60c5\u51b5\uff0c\u53ef\u4ee5\u9009\u62e9\u8bbe\u7f6e\u4e3a100\u81f3300\u4e4b\u95f4\u7684\u503c\u3002<br \/>\n200<br \/>\nhbase.hregion.max.filesize<br \/>\n\u8868\u793aHBase\u4e2dRegion\u7684\u6587\u4ef6\u603b\u5927\u5c0f\u7684\u6700\u5927\u503c\u3002\u5f53Region\u4e2d\u7684\u6587\u4ef6\u5927\u4e8e\u8be5\u53c2\u6570\u65f6\uff0c\u5c06\u4f1a\u5bfc\u81f4Region\u5206\u88c2\u3002 \u8be5\u53c2\u6570\u8bbe\u7f6e\u8fc7\u5c0f\u65f6\uff0c\u53ef\u80fd\u4f1a\u5bfc\u81f4Split\u64cd\u4f5c\u8fc7\u4e8e\u9891\u7e41\u3002\u5f53\u8bbe\u7f6e\u8fc7\u5927\u65f6\uff0c\u53ef\u80fd\u5bfc\u81f4Compact\u9700\u8981\u5904\u7406\u7684\u6587\u4ef6\u5927\u5c0f\u589e\u52a0\uff0c\u5f71\u54cdCompact\u6267\u884c\u6548\u7387\u3002<br \/>\n10737418240\uff08\u5355\u4f4d\uff1a\u5b57\u8282\uff09<br \/>\nhbase.hregion.memstore.flush.size<br \/>\n\u5728RegionServer\u4e2d\uff0c\u5f53\u5199\u64cd\u4f5c\u5185\u5b58\u4e2d\u5b58\u5728\u8d85\u8fc7memstore.flush.size\u5927\u5c0f\u7684memstore\uff0c\u5219MemStoreFlusher\u5c31\u542f\u52a8flush\u64cd\u4f5c\u5c06\u8be5memstore\u4ee5hfile\u7684\u5f62\u5f0f\u5199\u5165\u5bf9\u5e94\u7684store\u4e2d\u3002 \u5982\u679cRegionServer\u7684\u5185\u5b58\u5145\u8db3\uff0c\u800c\u4e14\u6d3b\u8dc3Region\u6570\u91cf\u4e5f\u4e0d\u662f\u5f88\u591a\u7684\u65f6\u5019\uff0c\u53ef\u4ee5\u9002\u5f53\u589e\u5927\u8be5\u503c\uff0c\u53ef\u4ee5\u51cf\u5c11compaction\u7684\u6b21\u6570\uff0c\u6709\u52a9\u4e8e\u63d0\u5347\u7cfb\u7edf\u6027\u80fd\u3002 \u540c\u65f6\uff0c\u8fd9\u79cdflush\u4ea7\u751f\u7684\u65f6\u5019\uff0c\u5e76\u4e0d\u662f\u7d27\u6025\u7684flush\uff0cflush\u64cd\u4f5c\u53ef\u80fd\u4f1a\u6709\u4e00\u5b9a\u5ef6\u8fdf\uff0c\u5728\u5ef6\u8fdf\u671f\u95f4\uff0c\u5199\u64cd\u4f5c\u8fd8\u53ef\u4ee5\u8fdb\u884c\uff0cMemstore\u8fd8\u4f1a\u7ee7\u7eed\u589e\u5927\uff0c\u6700\u5927\u503c\u4e3a\u201cmemstore.flush.size\u201d * \u201chbase.hregion.memstore.block.multiplier\u201d\u3002\u5f53\u8d85\u8fc7\u6700\u5927\u503c\u65f6\uff0c\u5c06\u4f1a\u963b\u585e\u64cd\u4f5c\u3002\u9002\u5f53\u589e\u5927\u201chbase.hregion.memstore.block.multiplier\u201d\u53ef\u4ee5\u51cf\u5c11\u963b\u585e\uff0c\u51cf\u5c11\u6027\u80fd\u6ce2\u52a8\u3002<br \/>\n134217728\uff08\u5355\u4f4d\uff1a\u5b57\u8282\uff09<br \/>\nhbase.regionserver.global.memstore.size<br \/>\nRegionServer\u4e2d\uff0c\u8d1f\u8d23flush\u64cd\u4f5c\u7684\u662fMemStoreFlusher\u7ebf\u7a0b\u3002\u8be5\u7ebf\u7a0b\u5b9a\u671f\u68c0\u67e5\u5199\u64cd\u4f5c\u5185\u5b58\uff0c\u5f53\u5199\u64cd\u4f5c\u5360\u7528\u5185\u5b58\u603b\u91cf\u8fbe\u5230\u9608\u503c\uff0cMemStoreFlusher\u5c06\u542f\u52a8flush\u64cd\u4f5c\uff0c\u6309\u7167\u4ece\u5927\u5230\u5c0f\u7684\u987a\u5e8f\uff0cflush\u82e5\u5e72\u76f8\u5bf9\u8f83\u5927\u7684memstore\uff0c\u76f4\u5230\u6240\u5360\u7528\u5185\u5b58\u5c0f\u4e8e\u9608\u503c\u3002 \u9608\u503c = \u201chbase.regionserver.global.memstore.size\u201d * \u201chbase.regionserver.global.memstore.size.lower.limit\u201d * \u201cHBase_HEAPSIZE\u201d \u8bf4\u660e\uff1a \u8be5\u914d\u7f6e\u4e0e\u201chfile.block.cache.size\u201d\u7684\u548c\u4e0d\u80fd\u8d85\u8fc70.8\uff0c\u4e5f\u5c31\u662f\u5199\u548c\u8bfb\u64cd\u4f5c\u7684\u5185\u5b58\u4e0d\u80fd\u8d85\u8fc7HeapSize\u768480%\uff0c\u8fd9\u6837\u53ef\u4ee5\u4fdd\u8bc1\u9664\u8bfb\u548c\u5199\u5916\u5176\u5b83\u64cd\u4f5c\u7684\u6b63\u5e38\u8fd0\u884c\u3002<br \/>\n0.4<br \/>\nhbase.hstore.blockingStoreFiles<br \/>\n\u5728region flush\u524d\u9996\u5148\u5224\u65adfile\u6587\u4ef6\u4e2a\u6570\uff0c\u662f\u5426\u5927\u4e8ehbase.hstore.blockingStoreFiles\u3002 \u5982\u679c\u5927\u4e8e\u9700\u8981\u5148compaction\u5e76\u4e14\u8ba9flush\u5ef6\u65f690s\uff08\u8fd9\u4e2a\u503c\u53ef\u4ee5\u901a\u8fc7hbase.hstore.blockingWaitTime\u8fdb\u884c\u914d\u7f6e\uff09\uff0c\u5728\u5ef6\u65f6\u8fc7\u7a0b\u4e2d\uff0c\u5c06\u4f1a\u7ee7\u7eed\u5199\u4ece\u800c\u4f7f\u5f97Memstore\u8fd8\u4f1a\u7ee7\u7eed\u589e\u5927\u8d85\u8fc7\u6700\u5927\u503c \u201cmemstore.flush.size\u201d * \u201chbase.hregion.memstore.block.multiplier\u201d\uff0c\u5bfc\u81f4\u5199\u64cd\u4f5c\u963b\u585e\u3002\u5f53\u5b8c\u6210compaction\u540e\uff0c\u53ef\u80fd\u5c31\u4f1a\u4ea7\u751f\u5927\u91cf\u5199\u5165\u3002\u8fd9\u6837\u5c31\u5bfc\u81f4\u6027\u80fd\u6fc0\u70c8\u9707\u8361\u3002 \u589e\u52a0hbase.hstore.blockingStoreFiles\uff0c\u53ef\u4ee5\u51cf\u4f4eBLOCK\u51e0\u7387\u3002<br \/>\n15<br \/>\nhbase.regionserver.thread.compaction.throttle<br \/>\n\u63a7\u5236\u4e00\u6b21Minor Compaction\u65f6\uff0c\u8fdb\u884ccompaction\u7684\u6587\u4ef6\u603b\u5927\u5c0f\u7684\u9608\u503c\u3002Compaction\u65f6\u7684\u6587\u4ef6\u603b\u5927\u5c0f\u4f1a\u5f71\u54cd\u8fd9\u4e00\u6b21compaction\u7684\u6267\u884c\u65f6\u95f4\uff0c\u5982\u679c\u592a\u5927\uff0c\u53ef\u80fd\u4f1a\u963b\u585e\u5176\u5b83\u7684compaction\u6216flush\u64cd\u4f5c\u3002<br \/>\n1610612736\uff08\u5355\u4f4d\uff1a\u5b57\u8282\uff09<br \/>\nhbase.hstore.compaction.min<br \/>\n\u5f53\u4e00\u4e2aStore\u4e2d\u6587\u4ef6\u8d85\u8fc7\u8be5\u503c\u65f6\uff0c\u4f1a\u8fdb\u884ccompact\uff0c\u9002\u5f53\u589e\u5927\u8be5\u503c\uff0c\u53ef\u4ee5\u51cf\u5c11\u6587\u4ef6\u88ab\u91cd\u590d\u6267\u884ccompaction\u3002\u4f46\u662f\u5982\u679c\u8fc7\u5927\uff0c\u4f1a\u5bfc\u81f4Store\u4e2d\u6587\u4ef6\u6570\u8fc7\u591a\u800c\u5f71\u54cd\u8bfb\u53d6\u7684\u6027\u80fd\u3002<br \/>\n6<br \/>\nhbase.hstore.compaction.max<br \/>\n\u63a7\u5236\u4e00\u6b21compaction\u64cd\u4f5c\u65f6\u7684\u6587\u4ef6\u6570\u91cf\u7684\u6700\u5927\u503c\u3002\u4e0e\u201chbase.hstore.compaction.max.size\u201d\u7684\u4f5c\u7528\u57fa\u672c\u76f8\u540c\uff0c\u4e3b\u8981\u662f\u63a7\u5236\u4e00\u6b21compaction\u64cd\u4f5c\u7684\u65f6\u95f4\u4e0d\u8981\u592a\u957f\u3002<br \/>\n10<br \/>\nhbase.hstore.compaction.max.size<br \/>\n\u5982\u679c\u4e00\u4e2aHFile\u6587\u4ef6\u7684\u5927\u5c0f\u5927\u4e8e\u8be5\u503c\uff0c\u90a3\u4e48\u5728Minor Compaction\u64cd\u4f5c\u4e2d\u4e0d\u4f1a\u9009\u62e9\u8fd9\u4e2a\u6587\u4ef6\u8fdb\u884ccompaction\u64cd\u4f5c\uff0c\u9664\u975e\u8fdb\u884cMajor Compaction\u64cd\u4f5c\u3002 \u8fd9\u4e2a\u503c\u53ef\u4ee5\u9632\u6b62\u8f83\u5927\u7684HFile\u53c2\u4e0ecompaction\u64cd\u4f5c\u3002\u5728\u7981\u6b62Major Compaction\u540e\uff0c\u4e00\u4e2aStore\u4e2d\u53ef\u80fd\u5b58\u5728\u51e0\u4e2aHFile\uff0c\u800c\u4e0d\u4f1a\u5408\u5e76\u6210\u4e3a\u4e00\u4e2aHFile\uff0c\u8fd9\u6837\u4e0d\u4f1a\u5bf9\u6570\u636e\u8bfb\u53d6\u9020\u6210\u592a\u5927\u7684\u6027\u80fd\u5f71\u54cd\u3002<br \/>\n9223372036854775807\uff08\u5355\u4f4d\uff1a\u5b57\u8282\uff09<br \/>\nhbase.hregion.majorcompaction<br \/>\n\u8bbe\u7f6eMajor Compaction\u7684\u6267\u884c\u5468\u671f\u3002\u9ed8\u8ba4\u503c\u4e3a604800000\u6beb\u79d2\u3002\u7531\u4e8e\u6267\u884cMajor Compaction\u4f1a\u5360\u7528\u8f83\u591a\u7684\u7cfb\u7edf\u8d44\u6e90\uff0c\u5982\u679c\u6b63\u5728\u5904\u4e8e\u7cfb\u7edf\u7e41\u5fd9\u65f6\u671f\uff0c\u4f1a\u5f71\u54cd\u7cfb\u7edf\u7684\u6027\u80fd\u3002 \u5982\u679c\u4e1a\u52a1\u6ca1\u6709\u8f83\u591a\u7684\u66f4\u65b0\u3001\u5220\u9664\u3001\u56de\u6536\u8fc7\u671f\u6570\u636e\u7a7a\u95f4\u65f6\uff0c\u53ef\u4ee5\u628a\u8be5\u503c\u8bbe\u7f6e\u4e3a0\uff0c\u4ee5\u7981\u6b62Major Compaction\u3002 \u5982\u679c\u5fc5\u987b\u8981\u6267\u884cMajor Compaction\uff0c\u4ee5\u56de\u6536\u66f4\u591a\u7684\u7a7a\u95f4\uff0c\u53ef\u4ee5\u9002\u5f53\u589e\u52a0\u8be5\u503c\uff0c\u540c\u65f6\u914d\u7f6e\u53c2\u6570\u201chbase.offpeak.end.hour\u201d\u548c\u201chbase.offpeak.start.hour\u201d\u4ee5\u63a7\u5236Major Compaction\u53d1\u751f\u5728\u4e1a\u52a1\u7a7a\u95f2\u7684\u65f6\u671f\u3002<br \/>\n604800000\uff08\u5355\u4f4d\uff1a\u6beb\u79d2\uff09<br \/>\nhbase.regionserver.maxlogs hbase.regionserver.hlog.blocksize<br \/>\n\u8868\u793a\u4e00\u4e2aRegionServer\u4e0a\u672a\u8fdb\u884cFlush\u7684Hlog\u7684\u6587\u4ef6\u6570\u91cf\u7684\u9608\u503c\uff0c\u5982\u679c\u5927\u4e8e\u8be5\u503c\uff0cRegionServer\u4f1a\u5f3a\u5236\u8fdb\u884cflush\u64cd\u4f5c\u3002 \u8868\u793a\u6bcf\u4e2aHLog\u6587\u4ef6\u7684\u6700\u5927\u5927\u5c0f\u3002\u5982\u679cHLog\u6587\u4ef6\u5927\u5c0f\u5927\u4e8e\u8be5\u503c\uff0c\u5c31\u4f1a\u6eda\u52a8\u51fa\u4e00\u4e2a\u65b0\u7684HLog\u6587\u4ef6\uff0c\u65e7\u7684\u5c06\u88ab\u7981\u7528\u5e76\u5f52\u6863\u3002 \u8fd9\u4e24\u4e2a\u53c2\u6570\u5171\u540c\u51b3\u5b9a\u4e86RegionServer\u4e2d\u53ef\u4ee5\u5b58\u5728\u7684\u672a\u8fdb\u884cFlush\u7684hlog\u6570\u91cf\u3002\u5f53\u8fd9\u4e2a\u6570\u636e\u91cf\u5c0f\u4e8eMemStore\u7684\u603b\u5927\u5c0f\u7684\u65f6\u5019\uff0c\u4f1a\u51fa\u73b0\u7531\u4e8eHLog\u6587\u4ef6\u8fc7\u591a\u800c\u89e6\u53d1\u7684\u5f3a\u5236flush\u64cd\u4f5c\u3002\u8fd9\u4e2a\u65f6\u5019\u53ef\u4ee5\u9002\u5f53\u8c03\u6574\u8fd9\u4e24\u4e2a\u53c2\u6570\u7684\u5927\u5c0f\uff0c\u4ee5\u907f\u514d\u51fa\u73b0\u8fd9\u79cd\u5f3a\u5236flush\u7684\u60c5\u51b5\u3002<br \/>\n32 134217728\uff08\u5355\u4f4d\uff1a\u5b57\u8282\uff09<\/p>\n<ul>\n<li><strong>\u5199\u6570\u636e\u5ba2\u6237\u7aef\u8c03\u4f18<\/strong><\/li>\n<\/ul>\n<p>\u5199\u6570\u636e\u65f6\uff0c\u5728\u573a\u666f\u5141\u8bb8\u7684\u60c5\u51b5\u4e0b\uff0c\u6700\u597d\u4f7f\u7528Put List\u7684\u65b9\u5f0f\uff0c\u53ef\u4ee5\u6781\u5927\u7684\u63d0\u5347\u5199\u6027\u80fd\u3002\u6bcf\u4e00\u6b21Put\u7684List\u7684\u957f\u5ea6\uff0c\u9700\u8981\u7ed3\u5408\u5355\u6761Put\u7684\u5927\u5c0f\uff0c\u4ee5\u53ca\u5b9e\u9645\u73af\u5883\u7684\u4e00\u4e9b\u53c2\u6570\u8fdb\u884c\u8bbe\u5b9a\u3002\u5efa\u8bae\u5728\u9009\u5b9a\u4e4b\u524d\u5148\u505a\u4e00\u4e9b\u57fa\u7840\u7684\u6d4b\u8bd5\u3002<\/p>\n<ul>\n<li><strong>\u5199\u6570\u636e\u8868\u8bbe\u8ba1\u8c03\u4f18<\/strong>\u886812-7\u00a0\u5f71\u54cd\u5b9e\u65f6\u5199\u6570\u636e\u76f8\u5173\u53c2\u6570<\/li>\n<\/ul>\n<p>COMPRESSION<br \/>\n\u914d\u7f6e\u6570\u636e\u7684\u538b\u7f29\u7b97\u6cd5\uff0c\u8fd9\u91cc\u7684\u538b\u7f29\u662fHFile\u4e2dblock\u7ea7\u522b\u7684\u538b\u7f29\u3002\u5bf9\u4e8e\u53ef\u4ee5\u538b\u7f29\u7684\u6570\u636e\uff0c\u914d\u7f6e\u538b\u7f29\u7b97\u6cd5\u53ef\u4ee5\u6709\u6548\u51cf\u5c11\u78c1\u76d8\u7684IO\uff0c\u4ece\u800c\u8fbe\u5230\u63d0\u9ad8\u6027\u80fd\u7684\u76ee\u7684\u3002 \u8bf4\u660e\uff1a \u5e76\u975e\u6240\u6709\u6570\u636e\u90fd\u53ef\u4ee5\u8fdb\u884c\u6709\u6548\u538b\u7f29\u3002\u4f8b\u5982\u4e00\u5f20\u56fe\u7247\u7684\u6570\u636e\uff0c\u56e0\u4e3a\u56fe\u7247\u4e00\u822c\u5df2\u7ecf\u662f\u538b\u7f29\u540e\u7684\u6570\u636e\uff0c\u6240\u4ee5\u538b\u7f29\u6548\u679c\u6709\u9650\u3002 \u5e38\u7528\u7684\u538b\u7f29\u7b97\u6cd5\u662fSNAPPY\uff0c\u56e0\u4e3a\u5b83\u6709\u8f83\u597d\u7684Encoding\/Decoding\u901f\u5ea6\u548c\u53ef\u4ee5\u63a5\u53d7\u7684\u538b\u7f29\u7387\u3002<br \/>\nNONE<br \/>\nBLOCKSIZE<br \/>\n\u914d\u7f6eHFile\u4e2dblock\u5757\u7684\u5927\u5c0f\uff0c\u4e0d\u540c\u7684block\u5757\u5927\u5c0f\uff0c\u53ef\u4ee5\u5f71\u54cdHBase\u8bfb\u5199\u6570\u636e\u7684\u6548\u7387\u3002\u8d8a\u5927\u7684block\u5757\uff0c\u914d\u5408\u538b\u7f29\u7b97\u6cd5\uff0c\u538b\u7f29\u7684\u6548\u7387\u5c31\u8d8a\u597d\uff1b\u4f46\u662f\u7531\u4e8eHBase\u7684\u8bfb\u53d6\u6570\u636e\u662f\u4ee5block\u5757\u4e3a\u5355\u4f4d\u7684\uff0c\u6240\u4ee5\u8d8a\u5927\u7684block\u5757\uff0c\u5bf9\u4e8e\u968f\u673a\u8bfb\u7684\u60c5\u51b5\uff0c\u6027\u80fd\u53ef\u80fd\u4f1a\u6bd4\u8f83\u5dee\u3002 \u5982\u679c\u8981\u63d0\u5347\u5199\u5165\u7684\u6027\u80fd\uff0c\u4e00\u822c\u6269\u5927\u5230128KB\u6216\u8005256KB\uff0c\u53ef\u4ee5\u63d0\u5347\u5199\u6570\u636e\u7684\u6548\u7387\uff0c\u4e5f\u4e0d\u4f1a\u5f71\u54cd\u592a\u5927\u7684\u968f\u673a\u8bfb\u6027\u80fd\u3002<br \/>\n65536\uff08\u5355\u4f4d\uff1a\u5b57\u8282\uff09<br \/>\nIN_MEMORY<br \/>\n\u914d\u7f6e\u8fd9\u4e2a\u8868\u7684\u6570\u636e\u4f18\u5148\u7f13\u5b58\u5728\u5185\u5b58\u4e2d\uff0c\u8fd9\u6837\u53ef\u4ee5\u6709\u6548\u63d0\u5347\u8bfb\u53d6\u7684\u6027\u80fd\u3002\u5bf9\u4e8e\u4e00\u4e9b\u5c0f\u8868\uff0c\u800c\u4e14\u9700\u8981\u9891\u7e41\u8fdb\u884c\u8bfb\u53d6\u64cd\u4f5c\u7684\uff0c\u53ef\u4ee5\u8bbe\u7f6e\u6b64\u914d\u7f6e\u9879\u3002<br \/>\nfalse<\/p>\n<h3>12.3.5\u00a0\u63d0\u5347\u5b9e\u65f6\u8bfb\u6570\u636e\u6548\u7387<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u9700\u8981\u8bfb\u53d6HBase\u6570\u636e\u573a\u666f\u3002<\/p>\n<p>\u524d\u63d0\u6761\u4ef6<\/p>\n<p>\u8c03\u7528HBase\u7684get\u6216scan\u63a5\u53e3\uff0c\u4eceHBase\u4e2d\u5b9e\u65f6\u8bfb\u53d6\u6570\u636e\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ul>\n<li><strong>\u8bfb\u6570\u636e\u670d\u52a1\u7aef\u8c03\u4f18<\/strong><\/li>\n<\/ul>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; HBase &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<br \/>\n\u886812-8\u00a0\u5f71\u54cd\u5b9e\u65f6\u5199\u6570\u636e\u914d\u7f6e\u9879<\/p>\n<p>GC_OPTS<br \/>\nHBase\u5229\u7528\u5185\u5b58\u5b8c\u6210\u8bfb\u5199\u64cd\u4f5c\u3002\u63d0\u9ad8HBase\u5185\u5b58\u53ef\u4ee5\u6709\u6548\u63d0\u9ad8HBase\u6027\u80fd\u3002 GC_OPTS\u4e3b\u8981\u9700\u8981\u8c03\u6574HeapSize\u7684\u5927\u5c0f\u548cNewSize\u7684\u5927\u5c0f\u3002\u8c03\u6574HeapSize\u5927\u5c0f\u7684\u65f6\u5019\uff0c\u5efa\u8bae\u5c06Xms\u548cXmx\u8bbe\u7f6e\u6210\u76f8\u540c\u7684\u503c\uff0c\u8fd9\u6837\u53ef\u4ee5\u907f\u514dJVM\u52a8\u6001\u8c03\u6574HeapSize\u5927\u5c0f\u7684\u65f6\u5019\u5f71\u54cd\u6027\u80fd\u3002\u8c03\u6574NewSize\u5927\u5c0f\u7684\u65f6\u5019\uff0c\u5efa\u8bae\u628a\u5176\u8bbe\u7f6e\u4e3aHeapSize\u5927\u5c0f\u76841\/9\u3002 HMaster\uff1a\u5f53HBase\u96c6\u7fa4\u89c4\u6a21\u8d8a\u5927\u3001Region\u6570\u91cf\u8d8a\u591a\u65f6\uff0c\u53ef\u4ee5\u9002\u5f53\u8c03\u5927HMaster\u7684GC_OPTS\u53c2\u6570\u3002 RegionServer\uff1aRegionServer\u9700\u8981\u7684\u5185\u5b58\u4e00\u822c\u6bd4HMaster\u8981\u5927\u3002\u5728\u5185\u5b58\u5145\u8db3\u7684\u60c5\u51b5\u4e0b\uff0cHeapSize\u53ef\u4ee5\u76f8\u5bf9\u8bbe\u7f6e\u5927\u4e00\u4e9b\u3002 \u8bf4\u660e\uff1a \u4e3bHMaster\u7684HeapSize\u4e3a4G\u7684\u65f6\u5019\uff0cHBase\u96c6\u7fa4\u53ef\u4ee5\u652f\u6301100000Region\u6570\u7684\u89c4\u6a21\u3002\u6839\u636e\u7ecf\u9a8c\u503c\uff0c\u5355\u4e2aRegionServer\u7684HeapSize\u4e0d\u5efa\u8bae\u8d85\u8fc720GB\u3002<br \/>\nHMaster\uff1a -Xms2G -Xmx2G -XX:NewSize=256M -XX:MaxNewSize=256M RegionServer\uff1a -Xms4G -Xmx4G -XX:NewSize=512M -XX:MaxNewSize=512M<br \/>\nhbase.regionserver.handler.count<br \/>\n\u8868\u793aRegionServer\u5728\u540c\u4e00\u65f6\u523b\u80fd\u591f\u5e76\u53d1\u5904\u7406\u591a\u5c11\u8bf7\u6c42\u3002\u5982\u679c\u8bbe\u7f6e\u8fc7\u9ad8\u4f1a\u5bfc\u81f4\u6fc0\u70c8\u7ebf\u7a0b\u7ade\u4e89\uff0c\u5982\u679c\u8bbe\u7f6e\u8fc7\u5c0f\uff0c\u8bf7\u6c42\u5c06\u4f1a\u5728RegionServer\u957f\u65f6\u95f4\u7b49\u5f85\uff0c\u964d\u4f4e\u5904\u7406\u80fd\u529b\u3002\u6839\u636e\u8d44\u6e90\u60c5\u51b5\uff0c\u9002\u5f53\u589e\u52a0\u5904\u7406\u7ebf\u7a0b\u6570\u3002 \u5efa\u8bae\u6839\u636eCPU\u7684\u4f7f\u7528\u60c5\u51b5\uff0c\u53ef\u4ee5\u9009\u62e9\u8bbe\u7f6e\u4e3a100\u81f3300\u4e4b\u95f4\u7684\u503c\u3002<br \/>\n200<br \/>\nhfile.block.cache.size<br \/>\nHBase\u7f13\u5b58\u533a\u5927\u5c0f\uff0c\u4e3b\u8981\u5f71\u54cd\u67e5\u8be2\u6027\u80fd\u3002\u6839\u636e\u67e5\u8be2\u6a21\u5f0f\u4ee5\u53ca\u67e5\u8be2\u8bb0\u5f55\u5206\u5e03\u60c5\u51b5\u6765\u51b3\u5b9a\u7f13\u5b58\u533a\u7684\u5927\u5c0f\u3002\u5982\u679c\u91c7\u7528\u968f\u673a\u67e5\u8be2\u4f7f\u5f97\u7f13\u5b58\u533a\u7684\u547d\u4e2d\u7387\u8f83\u4f4e\uff0c\u53ef\u4ee5\u9002\u5f53\u964d\u4f4e\u7f13\u5b58\u533a\u5927\u5c0f\u3002<br \/>\n0.25\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u5982\u679c\u540c\u65f6\u5b58\u5728\u8bfb\u548c\u5199\u7684\u64cd\u4f5c\uff0c\u8fd9\u4e24\u79cd\u64cd\u4f5c\u7684\u6027\u80fd\u4f1a\u4e92\u76f8\u5f71\u54cd\u3002\u5982\u679c\u5199\u5165\u5bfc\u81f4\u7684flush\u548cCompaction\u64cd\u4f5c\u9891\u7e41\u53d1\u751f\uff0c\u4f1a\u5360\u7528\u5927\u91cf\u7684\u78c1\u76d8IO\u64cd\u4f5c\uff0c\u4ece\u800c\u5f71\u54cd\u8bfb\u53d6\u7684\u6027\u80fd\u3002\u5982\u679c\u5199\u5165\u5bfc\u81f4\u963b\u585e\u8f83\u591a\u7684Compaction\u64cd\u4f5c\uff0c\u5c31\u4f1a\u51fa\u73b0Region\u4e2d\u5b58\u5728\u591a\u4e2aHFile\u7684\u60c5\u51b5\uff0c\u4ece\u800c\u5f71\u54cd\u8bfb\u53d6\u7684\u6027\u80fd\u3002\u6240\u4ee5\u5982\u679c\u8bfb\u53d6\u7684\u6027\u80fd\u4e0d\u7406\u60f3\u7684\u65f6\u5019\uff0c\u4e5f\u8981\u8003\u8651\u5199\u5165\u7684\u914d\u7f6e\u662f\u5426\u5408\u7406\u3002<\/p>\n<ul>\n<li><strong>\u8bfb\u6570\u636e\u5ba2\u6237\u7aef\u8c03\u4f18<\/strong><\/li>\n<\/ul>\n<p>Scan\u6570\u636e\u65f6\u9700\u8981\u8bbe\u7f6ecaching\uff08\u4e00\u6b21\u4ece\u670d\u52a1\u7aef\u8bfb\u53d6\u7684\u8bb0\u5f55\u6761\u6570\uff0c\u9ed8\u8ba4\u662f1\uff09\uff0c\u82e5\u4f7f\u7528\u9ed8\u8ba4\u503c\u8bfb\u6027\u80fd\u4f1a\u964d\u5230\u6781\u4f4e\u3002<\/p>\n<p>\u5f53\u4e0d\u9700\u8981\u8bfb\u4e00\u6761\u6570\u636e\u6240\u6709\u7684\u5217\u65f6\uff0c\u9700\u8981\u6307\u5b9a\u8bfb\u53d6\u7684\u5217\uff0c\u4ee5\u51cf\u5c11\u7f51\u7edcIO\u3002<\/p>\n<p>\u53ea\u8bfb\u53d6RowKey\u65f6\uff0c\u53ef\u4ee5\u4e3aScan\u6dfb\u52a0\u4e00\u4e2a\u53ea\u8bfb\u53d6RowKey\u7684filter\uff08FirstKeyOnlyFilter\u6216KeyOnlyFilter\uff09\u3002<\/p>\n<ul>\n<li><strong>\u8bfb\u6570\u636e\u8868\u8bbe\u8ba1\u8c03\u4f18<\/strong>\u886812-9\u00a0\u5f71\u54cd\u5b9e\u65f6\u8bfb\u6570\u636e\u76f8\u5173\u53c2\u6570<\/li>\n<\/ul>\n<p>COMPRESSION<br \/>\n\u914d\u7f6e\u6570\u636e\u7684\u538b\u7f29\u7b97\u6cd5\uff0c\u8fd9\u91cc\u7684\u538b\u7f29\u662fHFile\u4e2dblock\u7ea7\u522b\u7684\u538b\u7f29\u3002\u5bf9\u4e8e\u53ef\u4ee5\u538b\u7f29\u7684\u6570\u636e\uff0c\u914d\u7f6e\u538b\u7f29\u7b97\u6cd5\u53ef\u4ee5\u6709\u6548\u51cf\u5c11\u78c1\u76d8\u7684IO\uff0c\u4ece\u800c\u8fbe\u5230\u63d0\u9ad8\u6027\u80fd\u7684\u76ee\u7684\u3002 \u8bf4\u660e\uff1a \u5e76\u975e\u6240\u6709\u6570\u636e\u90fd\u53ef\u4ee5\u8fdb\u884c\u6709\u6548\u538b\u7f29\u3002\u4f8b\u5982\u4e00\u5f20\u56fe\u7247\u7684\u6570\u636e\uff0c\u56e0\u4e3a\u56fe\u7247\u4e00\u822c\u5df2\u7ecf\u662f\u538b\u7f29\u540e\u7684\u6570\u636e\uff0c\u6240\u4ee5\u538b\u7f29\u6548\u679c\u6709\u9650\u3002 \u5e38\u7528\u7684\u538b\u7f29\u7b97\u6cd5\u662fSNAPPY\uff0c\u56e0\u4e3a\u5b83\u6709\u8f83\u597d\u7684Encoding\/Decoding\u901f\u5ea6\u548c\u53ef\u4ee5\u63a5\u53d7\u7684\u538b\u7f29\u7387\u3002<br \/>\nNONE<br \/>\nBLOCKSIZE<br \/>\n\u914d\u7f6eHFile\u4e2dblock\u5757\u7684\u5927\u5c0f\uff0c\u4e0d\u540c\u7684block\u5757\u5927\u5c0f\uff0c\u53ef\u4ee5\u5f71\u54cdHBase\u8bfb\u5199\u6570\u636e\u7684\u6548\u7387\u3002\u8d8a\u5927\u7684block\u5757\uff0c\u914d\u5408\u538b\u7f29\u7b97\u6cd5\uff0c\u538b\u7f29\u7684\u6548\u7387\u5c31\u8d8a\u597d\uff1b\u4f46\u662f\u7531\u4e8eHBase\u7684\u8bfb\u53d6\u6570\u636e\u662f\u4ee5block\u5757\u4e3a\u5355\u4f4d\u7684\uff0c\u6240\u4ee5\u8d8a\u5927\u7684block\u5757\uff0c\u5bf9\u4e8e\u968f\u673a\u8bfb\u7684\u60c5\u51b5\uff0c\u6027\u80fd\u53ef\u80fd\u4f1a\u6bd4\u8f83\u5dee\u3002 \u5982\u679c\u8981\u63d0\u5347\u5199\u5165\u7684\u6027\u80fd\uff0c\u4e00\u822c\u6269\u5927\u5230128KB\u6216\u8005256KB\uff0c\u53ef\u4ee5\u63d0\u5347\u5199\u6570\u636e\u7684\u6548\u7387\uff0c\u4e5f\u4e0d\u4f1a\u5f71\u54cd\u592a\u5927\u7684\u968f\u673a\u8bfb\u6027\u80fd\u3002<br \/>\n65536\uff08\u5355\u4f4d\uff1a\u5b57\u8282\uff09<br \/>\nDATA_BLOCK_ENCODING<br \/>\n\u914d\u7f6eHFile\u4e2dblock\u5757\u7684\u7f16\u7801\u65b9\u6cd5\u3002\u5f53\u4e00\u884c\u6570\u636e\u4e2d\u5b58\u5728\u591a\u5217\u65f6\uff0c\u4e00\u822c\u53ef\u4ee5\u914d\u7f6e\u4e3a\u201cFAST_DIFF\u201d\uff0c\u53ef\u4ee5\u6709\u6548\u7684\u8282\u7701\u6570\u636e\u5b58\u50a8\u7684\u7a7a\u95f4\uff0c\u4ece\u800c\u63d0\u4f9b\u6027\u80fd\u3002<br \/>\nNONE<\/p>\n<h3>12.3.6\u00a0JVM\u53c2\u6570\u4f18\u5316<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5f53\u96c6\u7fa4\u6570\u636e\u91cf\u8fbe\u5230\u4e00\u5b9a\u89c4\u6a21\u540e\uff0cJVM\u7684\u9ed8\u8ba4\u914d\u7f6e\u5c06\u65e0\u6cd5\u6ee1\u8db3\u96c6\u7fa4\u7684\u4e1a\u52a1\u9700\u6c42\uff0c\u8f7b\u5219\u96c6\u7fa4\u53d8\u6162\uff0c\u91cd\u5219\u96c6\u7fa4\u670d\u52a1\u4e0d\u53ef\u7528\u3002\u6240\u4ee5\u9700\u8981\u6839\u636e\u5b9e\u9645\u7684\u4e1a\u52a1\u60c5\u51b5\u8fdb\u884c\u5408\u7406\u7684JVM\u53c2\u6570\u914d\u7f6e\uff0c\u63d0\u9ad8\u96c6\u7fa4\u6027\u80fd\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p><strong>\u53c2\u6570\u5165\u53e3\uff1a<\/strong><\/p>\n<p>HBase\u89d2\u8272\u76f8\u5173\u7684JVM\u53c2\u6570\u9700\u8981\u914d\u7f6e\u5728\u201c${HBASE_HOME}\/conf\u201d\u76ee\u5f55\u4e0b\u7684\u201chbase-env.sh\u201d\u6587\u4ef6\u4e2d\u3002<\/p>\n<p>\u6bcf\u4e2a\u89d2\u8272\u90fd\u6709\u5404\u81ea\u7684JVM\u53c2\u6570\u914d\u7f6e\u53d8\u91cf\uff0c\u5982\u886812-10\u3002<\/p>\n<p>\u886812-10\u00a0HBase\u76f8\u5173JVM\u53c2\u6570\u914d\u7f6e\u53d8\u91cf<\/p>\n<p>HBASE_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHBase\u7684\u6240\u6709\u89d2\u8272\u3002<br \/>\nSERVER_GC_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHBase Server\u7aef\u7684\u6240\u6709\u89d2\u8272\uff0c\u4f8b\u5982\uff1aMaster\u3001RegionServer\u7b49\u3002<br \/>\nCLIENT_GC_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHBase\u7684Client\u8fdb\u7a0b\u3002<br \/>\nHBASE_MASTER_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHBase\u7684Master\u3002<br \/>\nHBASE_REGIONSERVER_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHBase\u7684RegionServer\u3002<br \/>\nHBASE_THRIFT_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHBase\u7684Thrift\u3002<\/p>\n<p><strong>\u914d\u7f6e\u65b9\u5f0f\u4e3e\u4f8b\uff1a<\/strong><\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2export HADOOP_NAMENODE_OPTS=&amp;quot;-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS&amp;quot;<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<h2>12.4\u00a0HDFS<\/h2>\n<h3>12.4.1\u00a0\u63d0\u5347\u5199\u6027\u80fd<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5728HDFS\u4e2d\uff0c\u901a\u8fc7\u8c03\u6574\u5c5e\u6027\u7684\u503c\uff0c\u4f7f\u5f97HDFS\u96c6\u7fa4\u66f4\u9002\u5e94\u81ea\u8eab\u7684\u4e1a\u52a1\u60c5\u51b5\uff0c\u4ece\u800c\u63d0\u5347HDFS\u7684\u5199\u6027\u80fd\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; HDFS &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<\/p>\n<p>\u886812-11\u00a0HDFS\u5199\u6027\u80fd\u4f18\u5316\u914d\u7f6e<\/p>\n<p>dfs.datanode.drop.cache.behind.reads<br \/>\n\u8bbe\u7f6e\u4e3atrue\u8868\u793a\u4e22\u5f03\u7f13\u5b58\u7684\u6570\u636e\uff08\u9700\u8981\u5728DataNode\u4e2d\u914d\u7f6e\uff09\u3002 \u5f53\u540c\u4e00\u4efd\u6570\u636e\uff0c\u91cd\u590d\u8bfb\u53d6\u7684\u6b21\u6570\u8f83\u5c11\u65f6\uff0c\u5efa\u8bae\u8bbe\u7f6e\u4e3atrue\uff0c\u4f7f\u5f97\u7f13\u5b58\u80fd\u591f\u88ab\u5176\u4ed6\u64cd\u4f5c\u4f7f\u7528\u3002\u91cd\u590d\u8bfb\u53d6\u7684\u6b21\u6570\u8f83\u591a\u65f6\uff0c\u8bbe\u7f6e\u4e3afalse\u80fd\u591f\u63d0\u5347\u91cd\u590d\u8bfb\u53d6\u7684\u901f\u5ea6\u3002<br \/>\ntrue<br \/>\ndfs.client-write-packet-size<br \/>\n\u5f53HDFS Client\u5f80DataNode\u5199\u6570\u636e\u65f6\uff0c\u5c06\u6570\u636e\u751f\u6210\u4e00\u4e2a\u5305\u3002\u7136\u540e\u5c06\u8fd9\u4e2a\u5305\u5728\u7f51\u7edc\u4e0a\u4f20\u51fa\u3002\u6b64\u53c2\u6570\u6307\u5b9a\u4f20\u8f93\u6570\u636e\u5305\u7684\u5927\u5c0f\uff0c\u53ef\u4ee5\u901a\u8fc7\u5404Job\u6765\u6307\u5b9a\u3002\u5355\u4f4d\uff1a\u5b57\u8282\u3002 \u5728\u4e07\u5146\u7f51\u90e8\u7f72\u4e0b\uff0c\u53ef\u9002\u5f53\u589e\u5927\u8be5\u53c2\u6570\u503c\uff0c\u6765\u63d0\u5347\u4f20\u8f93\u7684\u541e\u5410\u91cf\u3002<br \/>\n262144<\/p>\n<h3>12.4.2\u00a0JVM\u53c2\u6570\u4f18\u5316<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5f53\u96c6\u7fa4\u6570\u636e\u91cf\u8fbe\u5230\u4e00\u5b9a\u89c4\u6a21\u540e\uff0cJVM\u7684\u9ed8\u8ba4\u914d\u7f6e\u5c06\u65e0\u6cd5\u6ee1\u8db3\u96c6\u7fa4\u7684\u4e1a\u52a1\u9700\u6c42\uff0c\u8f7b\u5219\u96c6\u7fa4\u53d8\u6162\uff0c\u91cd\u5219\u96c6\u7fa4\u670d\u52a1\u4e0d\u53ef\u7528\u3002\u6240\u4ee5\u9700\u8981\u6839\u636e\u5b9e\u9645\u7684\u4e1a\u52a1\u60c5\u51b5\u8fdb\u884c\u5408\u7406\u7684JVM\u53c2\u6570\u914d\u7f6e\uff0c\u63d0\u9ad8\u96c6\u7fa4\u6027\u80fd\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p><strong>\u53c2\u6570\u5165\u53e3\uff1a<\/strong><\/p>\n<p>HDFS\u89d2\u8272\u76f8\u5173\u7684JVM\u53c2\u6570\u9700\u8981\u914d\u7f6e\u5728\u201c${HADOOP_HOME}\/etc\/hadoop\u201d\u76ee\u5f55\u4e0b\u7684\u201chadoop-env.sh\u201d\u6587\u4ef6\u4e2d\u3002<\/p>\n<p>JVM\u5404\u53c2\u6570\u7684\u542b\u4e49\u8bf7\u53c2\u89c1\u5176\u5b98\u7f51\uff1a<a href=\"http:\/\/docs.oracle.com\/javase\/8\/docs\/technotes\/tools\/unix\/java.html\">http:\/\/docs.oracle.com\/javase\/8\/docs\/technotes\/tools\/unix\/java.html<\/a><\/p>\n<p>\u6bcf\u4e2a\u89d2\u8272\u90fd\u6709\u5404\u81ea\u7684JVM\u53c2\u6570\u914d\u7f6e\u53d8\u91cf\uff0c\u5982\u886812-12\u3002<\/p>\n<p>\u886812-12\u00a0HDFS\u76f8\u5173JVM\u53c2\u6570\u914d\u7f6e\u53d8\u91cf<\/p>\n<p>HADOOP_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHDFS\u7684\u6240\u6709\u89d2\u8272\u3002<br \/>\nHADOOP_NAMENODE_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHDFS\u7684NameNode\u3002<br \/>\nHADOOP_DATANODE_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHDFS\u7684DataNode\u3002<br \/>\nHADOOP_JOURNALNODE_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHDFS\u7684JournalNode\u3002<br \/>\nHADOOP_ZKFC_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHDFS\u7684ZKFC\u3002<br \/>\nHADOOP_SECONDARYNAMENODE_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHDFS\u7684SecondaryNameNode\u3002<br \/>\nHADOOP_CLIENT_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHDFS\u7684Client\u8fdb\u7a0b\u3002<br \/>\nHADOOP_BALANCER_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHDFS\u7684Balancer\u8fdb\u7a0b\u3002<br \/>\nHADOOP_MOVER_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdHDFS\u7684Mover\u8fdb\u7a0b\u3002<\/p>\n<p><strong>\u914d\u7f6e\u65b9\u5f0f\u4e3e\u4f8b\uff1a<\/strong><\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2export HADOOP_NAMENODE_OPTS=&amp;quot;-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS&amp;quot;<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<h3>12.4.3\u00a0\u4f7f\u7528\u5ba2\u6237\u7aef\u5143\u6570\u636e\u7f13\u5b58\u63d0\u9ad8\u8bfb\u53d6\u6027\u80fd<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u901a\u8fc7\u4f7f\u7528\u5ba2\u6237\u7aef\u7f13\u5b58\u5143\u6570\u636e\u5757\u7684\u4f4d\u7f6e\u6765\u63d0\u9ad8HDFS\u8bfb\u53d6\u6027\u80fd\u3002<\/p>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u6b64\u529f\u80fd\u4ec5\u7528\u4e8e\u8bfb\u53d6\u4e0d\u7ecf\u5e38\u4fee\u6539\u7684\u6587\u4ef6\u3002\u56e0\u4e3a\u5728\u670d\u52a1\u5668\u7aef\u7531\u67d0\u4e9b\u5176\u4ed6\u5ba2\u6237\u7aef\u5b8c\u6210\u7684\u6570\u636e\u4fee\u6539\uff0c\u5bf9\u4e8e\u9ad8\u901f\u7f13\u5b58\u7684\u5ba2\u6237\u7aef\u5c06\u662f\u4e0d\u53ef\u89c1\u7684\uff0c\u8fd9\u53ef\u80fd\u5bfc\u81f4\u4ece\u7f13\u5b58\u4e2d\u62ff\u5230\u7684\u5143\u6570\u636e\u662f\u8fc7\u671f\u7684\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u8bbe\u7f6e\u53c2\u6570\u7684\u8def\u5f84\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u9875\u9762\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406\u00a0&gt;\u00a0HDFS\u00a0&gt;\u00a0\u670d\u52a1\u914d\u7f6e\u201d\uff0c\u5c06\u201c\u53c2\u6570\u7c7b\u522b\u201d\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\uff0c\u5e76\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<\/p>\n<p>\u886812-13\u00a0\u53c2\u6570\u914d\u7f6e<\/p>\n<p>dfs.client.metadata.cache.enabled<br \/>\n\u542f\u7528\/\u7981\u7528\u5757\u4f4d\u7f6e\u5143\u6570\u636e\u7684\u5ba2\u6237\u7aef\u7f13\u5b58\u3002\u5c06\u6b64\u53c2\u6570\u8bbe\u7f6e\u4e3a\u201ctrue\u201d\uff0c\u642d\u914d\u201cdfs.client.metadata.cache.pattern\u201d\u53c2\u6570\u4ee5\u542f\u7528\u7f13\u5b58\u3002<br \/>\nfalse<br \/>\ndfs.client.metadata.cache.pattern<br \/>\n\u9700\u8981\u7f13\u5b58\u7684\u6587\u4ef6\u8def\u5f84\u7684\u6b63\u5219\u8868\u8fbe\u5f0f\u6a21\u5f0f\u3002\u53ea\u6709\u8fd9\u4e9b\u6587\u4ef6\u7684\u5757\u4f4d\u7f6e\u5143\u6570\u636e\u88ab\u7f13\u5b58\uff0c\u76f4\u5230\u8fd9\u4e9b\u5143\u6570\u636e\u8fc7\u671f\u3002\u6b64\u914d\u7f6e\u4ec5\u5728\u53c2\u6570\u201cdfs.client.metadata.cache.enabled\u201d\u8bbe\u7f6e\u4e3a\u201ctrue\u201d\u65f6\u6709\u6548\u3002 \u793a\u4f8b\uff1a\u201c\/test.*\u201d\u8868\u793a\u8bfb\u53d6\u5176\u8def\u5f84\u662f\u4ee5\u201c\/test\u201d\u5f00\u5934\u7684\u6240\u6709\u6587\u4ef6\u3002 \u8bf4\u660e\uff1a \u4e3a\u786e\u4fdd\u4e00\u81f4\u6027\uff0c\u914d\u7f6e\u7279\u5b9a\u6a21\u5f0f\u4ee5\u4ec5\u7f13\u5b58\u5176\u4ed6\u5ba2\u6237\u7aef\u4e0d\u7ecf\u5e38\u4fee\u6539\u7684\u6587\u4ef6\u3002 \u6b63\u5219\u8868\u8fbe\u5f0f\u6a21\u5f0f\u5c06\u4ec5\u9a8c\u8bc1URI\u7684path\u90e8\u5206\uff0c\u800c\u4e0d\u9a8c\u8bc1\u5728Fully Qualified\u8def\u5f84\u60c5\u51b5\u4e0b\u7684schema\u548cauthority\u3002<br \/>\n<!-- -->&lt;empty&gt;<!-- --><br \/>\ndfs.client.metadata.cache.expiry.sec<br \/>\n\u7f13\u5b58\u5143\u6570\u636e\u7684\u6301\u7eed\u65f6\u95f4\u3002\u7f13\u5b58\u6761\u76ee\u5728\u8be5\u6301\u7eed\u65f6\u95f4\u8fc7\u671f\u540e\u5931\u6548\u3002\u5373\u4f7f\u5728\u7f13\u5b58\u8fc7\u7a0b\u4e2d\u7ecf\u5e38\u4f7f\u7528\u7684\u5143\u6570\u636e\u4e5f\u4f1a\u53d1\u751f\u5931\u6548\u3002 \u914d\u7f6e\u503c\u53ef\u91c7\u7528\u65f6\u95f4\u540e\u7f00s\/m\/h\u8868\u793a\uff0c\u5206\u522b\u8868\u793a\u79d2\uff0c\u5206\u949f\u548c\u5c0f\u65f6\u3002 \u8bf4\u660e\uff1a \u82e5\u5c06\u8be5\u53c2\u6570\u914d\u7f6e\u4e3a\u201c0s\u201d\uff0c\u5c06\u7981\u7528\u7f13\u5b58\u529f\u80fd\u3002<br \/>\n60s<br \/>\ndfs.client.metadata.cache.max.entries<br \/>\n\u7f13\u5b58\u4e00\u6b21\u6700\u591a\u53ef\u4fdd\u5b58\u7684\u975e\u8fc7\u671f\u6570\u636e\u6761\u76ee\u3002<br \/>\n65536<\/p>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u8981\u5728\u8fc7\u671f\u524d\u5b8c\u5168\u6e05\u9664\u5ba2\u6237\u7aef\u7f13\u5b58\uff0c\u53ef\u8c03\u7528<em>DFSClient#clearLocatedBlockCache()<\/em>\u3002<\/p>\n<p>\u7528\u6cd5\u5982\u4e0b\u6240\u793a\u3002<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/>6<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2 &nbsp; &nbsp;FileSystem fs = FileSystem.get(conf);<br \/>\n3 &nbsp; &nbsp;DistributedFileSystem dfs = (DistributedFileSystem) fs;<br \/>\n4 &nbsp; &nbsp;DFSClient dfsClient = dfs.getClient();<br \/>\n5 &nbsp; &nbsp;dfsClient.clearLocatedBlockCache();<br \/>\n6<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<h3>12.4.4\u00a0\u4f7f\u7528\u5f53\u524d\u6d3b\u52a8\u7f13\u5b58\u63d0\u5347\u5ba2\u6237\u7aef\u4e0eNameNode\u7684\u8fde\u63a5\u6027\u80fd<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>HDFS\u90e8\u7f72\u5728\u5177\u6709\u591a\u4e2aNameNode\u5b9e\u4f8b\u7684HA\uff08High Availability\uff09\u6a21\u5f0f\u4e2d\uff0cHDFS\u5ba2\u6237\u7aef\u9700\u8981\u4f9d\u6b21\u8fde\u63a5\u5230\u6bcf\u4e2aNameNode\uff0c\u4ee5\u786e\u5b9a\u5f53\u524d\u6d3b\u52a8\u7684NameNode\u662f\u4ec0\u4e48\uff0c\u5e76\u5c06\u5176\u7528\u4e8e\u5ba2\u6237\u7aef\u64cd\u4f5c\u3002<\/p>\n<p>\u4e00\u65e6\u8bc6\u522b\u51fa\u6765\uff0c\u5f53\u524d\u6d3b\u52a8\u7684NameNode\u7684\u8be6\u7ec6\u4fe1\u606f\u5c31\u53ef\u4ee5\u88ab\u7f13\u5b58\u5e76\u5171\u4eab\u7ed9\u5728\u5ba2\u6237\u7aef\u673a\u5668\u4e2d\u8fd0\u884c\u7684\u6240\u6709\u5ba2\u6237\u7aef\u3002\u8fd9\u6837\uff0c\u6bcf\u4e2a\u65b0\u5ba2\u6237\u7aef\u53ef\u4ee5\u9996\u5148\u5c1d\u8bd5\u4ece\u7f13\u5b58\u52a0\u8f7d\u6d3b\u52a8\u7684Name Node\u7684\u8be6\u7ec6\u4fe1\u606f\uff0c\u5e76\u5c06RPC\u8c03\u7528\u4fdd\u5b58\u5230\u5907\u7528\u7684NameNode\u3002\u5728\u5f02\u5e38\u60c5\u51b5\u4e0b\u6709\u5f88\u591a\u4f18\u52bf\uff0c\u4f8b\u5982\u5f53\u5907\u7528\u7684NameNode\u8fde\u63a5\u957f\u65f6\u95f4\u4e0d\u54cd\u5e94\u65f6\u3002<\/p>\n<p>\u5f53\u53d1\u751f\u6545\u969c\uff0c\u5c06\u53e6\u4e00\u4e2aNameNode\u5207\u6362\u4e3a\u6d3b\u52a8\u72b6\u6001\u65f6\uff0c\u7f13\u5b58\u7684\u8be6\u7ec6\u4fe1\u606f\u5c06\u88ab\u66f4\u65b0\u4e3a\u5f53\u524d\u6d3b\u52a8\u7684NameNode\u7684\u4fe1\u606f\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u8bbe\u7f6e\u53c2\u6570\u7684\u8def\u5f84\u5982\u4e0b\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u9875\u9762\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; HDFS &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u5c06\u201c\u53c2\u6570\u7c7b\u522b\u201d\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\uff0c\u5e76\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<\/p>\n<p>\u886812-14\u00a0\u914d\u7f6e\u53c2\u6570<\/p>\n<p>dfs.client.failover.proxy.provider.[nameservice ID]<br \/>\n\u914d\u7f6e\u5ba2\u6237\u7aefFailover proxy provider\u7c7b\uff0c\u8be5\u7c7b\u4f7f\u7528\u4f20\u9012\u7684\u534f\u8bae\u521b\u5efaNameNode proxy\u3002\u8be5\u53c2\u6570\u53ef\u4ee5\u88ab\u914d\u7f6e\u4e3a\u201corg.apache.hadoop.hdfs.server.namenode.ha.BlackListingFailoverProxyProvider\u201d\u6216\u8005\u201corg.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\u201d\u3002<br \/>\norg.apache.hadoop.hdfs.server.namenode.ha.BlackListingFailoverProxyProvider<br \/>\ndfs.client.failover.activeinfo.share.flag<br \/>\n\u542f\u7528\u7f13\u5b58\u5e76\u5c06\u5f53\u524d\u6d3b\u52a8\u7684NameNode\u7684\u8be6\u7ec6\u4fe1\u606f\u5171\u4eab\u7ed9\u5176\u4ed6\u5ba2\u6237\u7aef\u3002\u82e5\u8981\u542f\u7528\u7f13\u5b58\uff0c\u9700\u5c06\u5176\u8bbe\u7f6e\u4e3a\u201ctrue\u201d\u3002<br \/>\nfalse<br \/>\ndfs.client.failover.activeinfo.share.path<br \/>\n\u6307\u5b9a\u5c06\u5728\u673a\u5668\u4e2d\u7684\u6240\u6709\u5ba2\u6237\u7aef\u521b\u5efa\u7684\u5171\u4eab\u6587\u4ef6\u7684\u672c\u5730\u76ee\u5f55\u3002\u5982\u679c\u8981\u4e3a\u4e0d\u540c\u7528\u6237\u5171\u4eab\u7f13\u5b58\uff0c\u8be5\u6587\u4ef6\u5939\u5e94\u5177\u6709\u5fc5\u9700\u7684\u6743\u9650\uff08\u5982\u5728\u7ed9\u5b9a\u76ee\u5f55\u4e2d\u521b\u5efa\uff0c\u8bfb\u5199\u7f13\u5b58\u6587\u4ef6\uff09\u3002<br \/>\n\/tmp<br \/>\ndfs.client.failover.activeinfo.share.io.timeout.sec<br \/>\n\u63a7\u5236\u8d85\u65f6\u7684\u53ef\u9009\u914d\u7f6e\u3002\u7528\u4e8e\u5728\u8bfb\u53d6\u6216\u5199\u5165\u7f13\u5b58\u6587\u4ef6\u65f6\u83b7\u53d6\u9501\u5b9a\u3002\u5982\u679c\u5728\u8be5\u65f6\u95f4\u5185\u65e0\u6cd5\u83b7\u53d6\u7f13\u5b58\u6587\u4ef6\u4e0a\u7684\u9501\u5b9a\uff0c\u5219\u653e\u5f03\u5c1d\u8bd5\u8bfb\u53d6\u6216\u66f4\u65b0\u7f13\u5b58\u3002\u5355\u4f4d\u4e3a\u79d2\u3002<br \/>\n5<\/p>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u7531HDFS\u5ba2\u6237\u7aef\u521b\u5efa\u7684\u7f13\u5b58\u6587\u4ef6\u5fc5\u987b\u7531\u5176\u4ed6\u5ba2\u6237\u7aef\u91cd\u65b0\u4f7f\u7528\u3002\u56e0\u6b64\uff0c\u8fd9\u4e9b\u6587\u4ef6\u6c38\u8fdc\u4e0d\u4f1a\u4ece\u672c\u5730\u7cfb\u7edf\u4e2d\u5220\u9664\u3002\u82e5\u7981\u7528\u8be5\u529f\u80fd\uff0c\u53ef\u80fd\u9700\u8981\u8fdb\u884c\u624b\u52a8\u6e05\u7406\u3002<\/p>\n<h2>12.5\u00a0Hive<\/h2>\n<h3>12.5.1\u00a0\u5efa\u7acb\u8868\u5206\u533a<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>Hive\u5728\u505aSelect\u67e5\u8be2\u65f6\uff0c\u4e00\u822c\u4f1a\u626b\u63cf\u6574\u4e2a\u8868\u5185\u5bb9\uff0c\u4f1a\u6d88\u8017\u8f83\u591a\u65f6\u95f4\u53bb\u626b\u63cf\u4e0d\u5173\u6ce8\u7684\u6570\u636e\u3002\u6b64\u65f6\uff0c\u53ef\u6839\u636e\u4e1a\u52a1\u9700\u6c42\u53ca\u5176\u67e5\u8be2\u7ef4\u5ea6\uff0c\u5efa\u7acb\u5408\u7406\u7684\u8868\u5206\u533a\uff0c\u4ece\u800c\u63d0\u9ad8\u67e5\u8be2\u6548\u7387\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ol>\n<li>\u4f7f\u7528PuTTY\u5de5\u5177\uff0c\u4ee5root\u7528\u6237\u767b\u5f55\u5df2\u5b89\u88c5Hive\u5ba2\u6237\u7aef\u7684\u8282\u70b9\u3002<\/li>\n<li>\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u8fdb\u5165\u5ba2\u6237\u7aef\u5b89\u88c5\u76ee\u5f55\uff0c\u4f8b\u5982\u201c\/opt\/client\u201d\u3002<\/li>\n<\/ol>\n<p><em><strong>cd \/opt\/client<\/strong><\/em><\/p>\n<ol>\n<li>\u6267\u884c<em><strong>source bigdata_env<\/strong><\/em>\u547d\u4ee4\uff0c\u914d\u7f6e\u5ba2\u6237\u7aef\u73af\u5883\u53d8\u91cf\u3002<\/li>\n<li>\u5728\u5ba2\u6237\u7aef\u4e2d\u6267\u884c\u5982\u4e0b\u547d\u4ee4\uff0c\u6267\u884c\u767b\u5f55\u64cd\u4f5c\u3002<\/li>\n<\/ol>\n<p>**<em>kinit\u00a0<strong>\u7528\u6237\u540d<\/strong><\/em><\/p>\n<ol>\n<li>\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u767b\u5f55\u5ba2\u6237\u7aef\u5de5\u5177\u3002<\/li>\n<\/ol>\n<p><em><strong>beeline<\/strong><\/em><\/p>\n<ol>\n<li>\u6307\u5b9a\u9759\u6001\u5206\u533a\u6216\u8005\u52a8\u6001\u5206\u533a\u3002<\/li>\n<\/ol>\n<ul>\n<li>\u9759\u6001\u5206\u533a\uff1a<\/li>\n<\/ul>\n<p>\u9759\u6001\u5206\u533a\u662f\u624b\u52a8\u8f93\u5165\u5206\u533a\u540d\u79f0\uff0c\u5728\u521b\u5efa\u8868\u65f6\u4f7f\u7528\u5173\u952e\u5b57<em><strong>PARTITIONED BY<\/strong><\/em>\u6307\u5b9a\u5206\u533a\u5217\u540d\u53ca\u6570\u636e\u7c7b\u578b\u3002\u5e94\u7528\u5f00\u53d1\u65f6\uff0c\u4f7f\u7528<em><strong>ALTER TABLE ADD PARTITION<\/strong><\/em>\u8bed\u53e5\u589e\u52a0\u5206\u533a\uff0c\u4ee5\u53ca\u4f7f\u7528<em><strong>LOAD DATA INTO PARTITON<\/strong><\/em>\u8bed\u53e5\u5c06\u6570\u636e\u52a0\u8f7d\u5230\u5206\u533a\u65f6\uff0c\u53ea\u80fd\u9759\u6001\u5206\u533a\u3002<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/><\/div><\/td><td><div class=\"text codecolorer\">1* \u52a8\u6001\u5206\u533a\uff1a\u901a\u8fc7\u67e5\u8be2\u547d\u4ee4\uff0c\u5c06\u7ed3\u679c\u63d2\u5165\u5230\u67d0\u4e2a\u8868\u7684\u5206\u533a\u65f6\uff0c\u53ef\u4ee5\u4f7f\u7528\u52a8\u6001\u5206\u533a\u3002 <br \/>\n2<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u52a8\u6001\u5206\u533a\u901a\u8fc7\u5728\u5ba2\u6237\u7aef\u5de5\u5177\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u6765\u5f00\u542f\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2set hive.exec.dynamic.partition=true<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u52a8\u6001\u5206\u533a\u9ed8\u8ba4\u6a21\u5f0f\u662fstrict\uff0c\u4e5f\u5c31\u662f\u5fc5\u987b\u81f3\u5c11\u6307\u5b9a\u4e00\u5217\u4e3a\u9759\u6001\u5206\u533a\uff0c\u5728\u9759\u6001\u5206\u533a\u4e0b\u5efa\u7acb\u52a8\u6001\u5b50\u5206\u533a\uff0c\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u8bbe\u7f6e\u6765\u5f00\u542f\u5b8c\u5168\u7684\u52a8\u6001\u5206\u533a\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2set hive.exec.dynamic.partition.mode=nonstrict<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<ol>\n<li>\u52a8\u6001\u5206\u533a\u53ef\u80fd\u5bfc\u81f4\u4e00\u4e2aDML\u8bed\u53e5\u521b\u5efa\u5927\u91cf\u7684\u5206\u533a\uff0c\u5bf9\u5e94\u7684\u521b\u5efa\u5927\u91cf\u65b0\u6587\u4ef6\u5939\uff0c\u5bf9\u7cfb\u7edf\u6027\u80fd\u53ef\u80fd\u5e26\u6765\u5f71\u54cd\u3002<\/li>\n<li>\u5728\u6587\u4ef6\u6570\u91cf\u5927\u7684\u60c5\u51b5\u4e0b\uff0c\u6267\u884c\u4e00\u4e2aSQL\u8bed\u53e5\u542f\u52a8\u65f6\u95f4\u8f83\u957f\uff0c\u53ef\u4ee5\u5728\u6267\u884cSQL\u8bed\u53e5\u4e4b\u524d\u6267\u884c\u201cset mapreduce.input.fileinputformat.list-status.num-threads = 100;\u201d\u8bed\u53e5\u6765\u7f29\u77ed\u542f\u52a8\u65f6\u95f4\u3002\u201cmapreduce.input.fileinputformat.list-status.num-threads\u201d\u53c2\u6570\u9700\u8981\u5148\u6dfb\u52a0\u5230Hive\u7684\u767d\u540d\u5355\u624d\u53ef\u8bbe\u7f6e\u3002<\/li>\n<\/ol>\n<h3>12.5.2\u00a0Join\u4f18\u5316<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u4f7f\u7528Join\u8bed\u53e5\u65f6\uff0c\u5982\u679c\u6570\u636e\u91cf\u5927\uff0c\u53ef\u80fd\u9020\u6210\u547d\u4ee4\u6267\u884c\u901f\u5ea6\u548c\u67e5\u8be2\u901f\u5ea6\u6162\uff0c\u6b64\u65f6\u53ef\u8fdb\u884cJoin\u4f18\u5316\u3002<\/p>\n<p>Join\u4f18\u5316\u53ef\u5206\u4e3a\u4ee5\u4e0b\u65b9\u5f0f\uff1a<\/p>\n<ul>\n<li>Map Join<\/li>\n<li>Sort Merge Bucket Map Join<\/li>\n<li>Join\u987a\u5e8f\u4f18\u5316<\/li>\n<\/ul>\n<p>Map Join<\/p>\n<p>Hive\u7684Map Join\u9002\u7528\u4e8e\u80fd\u591f\u5728\u5185\u5b58\u4e2d\u5b58\u653e\u4e0b\u7684\u5c0f\u8868\uff08\u6307\u8868\u5927\u5c0f\u5c0f\u4e8e25MB\uff09\uff0c\u901a\u8fc7\u201chive.mapjoin.smalltable.filesize\u201d\u5b9a\u4e49\u5c0f\u8868\u7684\u5927\u5c0f\uff0c\u9ed8\u8ba4\u4e3a25MB\u3002<\/p>\n<p>Map Join\u7684\u65b9\u6cd5\u6709\u4e24\u79cd\uff1a<\/p>\n<ul>\n<li>\n<p>\u4f7f\u7528\/*+ MAPJOIN(join_table) *\/\u3002<\/p>\n<\/li>\n<li>\n<p>\u6267\u884c\u8bed\u53e5\u524d\u8bbe\u7f6e\u5982\u4e0b\u53c2\u6570\uff0c\u5f53\u524d\u7248\u672c\u4e2d\u8be5\u503c\u9ed8\u8ba4\u4e3atrue\u3002<\/p>\n<\/li>\n<\/ul>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2set hive.auto.convert.join=true<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u4f7f\u7528Map Join\u65f6\u6ca1\u6709Reduce\u4efb\u52a1\uff0c\u800c\u662f\u5728Map\u4efb\u52a1\u524d\u8d77\u4e86\u4e00\u4e2aMapReduce Local Task\uff0c\u8fd9\u4e2aTask\u901a\u8fc7TableScan\u8bfb\u53d6\u5c0f\u8868\u5185\u5bb9\u5230\u672c\u673a\uff0c\u5728\u672c\u673a\u4ee5HashTable\u7684\u5f62\u5f0f\u4fdd\u5b58\u5e76\u5199\u5165\u786c\u76d8\u4e0a\u4f20\u5230DFS\uff0c\u5e76\u5728distributed cache\u4e2d\u4fdd\u5b58\uff0c\u5728Map Task\u4e2d\u4ece\u672c\u5730\u78c1\u76d8\u6216\u8005distributed cache\u4e2d\u8bfb\u53d6\u5c0f\u8868\u5185\u5bb9\u76f4\u63a5\u4e0e\u5927\u8868join\u5f97\u5230\u7ed3\u679c\u5e76\u8f93\u51fa\u3002<\/p>\n<p>\u4f7f\u7528Map Join\u65f6\u9700\u8981\u6ce8\u610f\u5c0f\u8868\u4e0d\u80fd\u8fc7\u5927\uff0c\u5982\u679c\u5c0f\u8868\u5c06\u5185\u5b58\u57fa\u672c\u7528\u5c3d\uff0c\u4f1a\u4f7f\u6574\u4e2a\u7cfb\u7edf\u6027\u80fd\u4e0b\u964d\u751a\u81f3\u51fa\u73b0\u5185\u5b58\u6ea2\u51fa\u7684\u5f02\u5e38\u3002<\/p>\n<p>Sort Merge Bucket Map Join<\/p>\n<p>\u4f7f\u7528Sort Merge Bucket Map Join\u5fc5\u987b\u6ee1\u8db3\u4ee5\u4e0b2\u4e2a\u6761\u4ef6\uff1a<\/p>\n<ol>\n<li>join\u7684\u4e24\u5f20\u8868\u90fd\u5f88\u5927\uff0c\u5185\u5b58\u4e2d\u65e0\u6cd5\u5b58\u653e\u3002<\/li>\n<li>\u4e24\u5f20\u8868\u90fd\u6309\u7167join key\u8fdb\u884c\u5206\u6876(clustered by (column))\u548c\u6392\u5e8f(sorted by(column))\uff0c\u4e14\u4e24\u5f20\u8868\u7684\u5206\u6876\u6570\u6b63\u597d\u662f\u500d\u6570\u5173\u7cfb\u3002<\/li>\n<\/ol>\n<p>\u901a\u8fc7\u5982\u4e0b\u8bbe\u7f6e\uff0c\u542f\u7528Sort Merge Bucket Map Join\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2set hive.optimize.bucketmapjoin=true <br \/>\n3<br \/>\n4 set hive.optimize.bucketmapjoin.sortedmerge=true<br \/>\n5<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u8fd9\u79cdMap Join\u4e5f\u6ca1\u6709Reduce\u4efb\u52a1\uff0c\u662f\u5728Map\u4efb\u52a1\u524d\u542f\u52a8MapReduce Local Task\uff0c\u5c06\u5c0f\u8868\u5185\u5bb9\u6309\u6876\u8bfb\u53d6\u5230\u672c\u5730\uff0c\u5728\u672c\u673a\u4fdd\u5b58\u591a\u4e2a\u6876\u7684HashTable\u5907\u4efd\u5e76\u5199\u5165HDFS\uff0c\u5e76\u4fdd\u5b58\u5728Distributed Cache\u4e2d\uff0c\u5728Map Task\u4e2d\u4ece\u672c\u5730\u78c1\u76d8\u6216\u8005Distributed Cache\u4e2d\u6309\u6876\u4e00\u4e2a\u4e00\u4e2a\u8bfb\u53d6\u5c0f\u8868\u5185\u5bb9\uff0c\u7136\u540e\u4e0e\u5927\u8868\u505a\u5339\u914d\u76f4\u63a5\u5f97\u5230\u7ed3\u679c\u5e76\u8f93\u51fa\u3002<\/p>\n<p>Join\u987a\u5e8f\u4f18\u5316<\/p>\n<p>\u5f53\u67093\u5f20\u53ca\u4ee5\u4e0a\u7684\u8868\u8fdb\u884cJoin\u65f6\uff0c\u9009\u62e9\u4e0d\u540c\u7684Join\u987a\u5e8f\uff0c\u6267\u884c\u65f6\u95f4\u5b58\u5728\u8f83\u5927\u5dee\u5f02\u3002\u4f7f\u7528\u6070\u5f53\u7684Join\u987a\u5e8f\u53ef\u4ee5\u6709\u6548\u7f29\u77ed\u4efb\u52a1\u6267\u884c\u65f6\u95f4\u3002<\/p>\n<p>Join\u987a\u5e8f\u539f\u5219\uff1a<\/p>\n<ul>\n<li>Join\u51fa\u6765\u7ed3\u679c\u8f83\u5c0f\u7684\u7ec4\u5408\uff0c\u4f8b\u5982\u8868\u6570\u636e\u91cf\u5c0f\u6216\u4e24\u5f20\u8868Join\u540e\u4ea7\u751f\u7ed3\u679c\u8f83\u5c11\uff0c\u4f18\u5148\u6267\u884c\u3002<\/li>\n<li>Join\u51fa\u6765\u7ed3\u679c\u5927\u7684\u7ec4\u5408\uff0c\u4f8b\u5982\u8868\u6570\u636e\u91cf\u5927\u6216\u4e24\u5f20\u8868Join\u540e\u4ea7\u751f\u7ed3\u679c\u8f83\u591a\uff0c\u5728\u540e\u9762\u6267\u884c\u3002<\/li>\n<\/ul>\n<p>\u4f8b\u5982\uff0ccustomer\u8868\u7684\u6570\u636e\u91cf\u6700\u591a\uff0corders\u8868\u548clineitem\u8868\u4f18\u5148Join\u53ef\u83b7\u5f97\u8f83\u5c11\u7684\u4e2d\u95f4\u7ed3\u679c\u3002<\/p>\n<p>\u539f\u6709\u7684Join\u8bed\u53e5\u5982\u4e0b\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/>6<br \/>7<br \/>8<br \/>9<br \/>10<br \/>11<br \/>12<br \/>13<br \/>14<br \/>15<br \/>16<br \/>17<br \/>18<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2select<br \/>\n3 &nbsp;l_orderkey,<br \/>\n4 &nbsp;sum(l_extendedprice * (1 - l_discount)) as revenue,<br \/>\n5 &nbsp;o_orderdate,<br \/>\n6 &nbsp;o_shippriority<br \/>\n7from<br \/>\n8 &nbsp;customer,<br \/>\n9 &nbsp;orders,<br \/>\n10 &nbsp;lineitem<br \/>\n11where<br \/>\n12 &nbsp;c_mktsegment = &amp;#x27;BUILDING&amp;#x27;<br \/>\n13 &nbsp;and c_custkey = o_custkey<br \/>\n14 &nbsp;and l_orderkey = o_orderkey<br \/>\n15 &nbsp;and o_orderdate &amp;lt; &amp;#x27;1995-03-22&amp;#x27;<br \/>\n16 &nbsp;and l_shipdate &amp;gt; &amp;#x27;1995-03-22&amp;#x27;<br \/>\n17limit 10;<br \/>\n18<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>Join\u987a\u5e8f\u4f18\u5316\u540e\u5982\u4e0b\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/>6<br \/>7<br \/>8<br \/>9<br \/>10<br \/>11<br \/>12<br \/>13<br \/>14<br \/>15<br \/>16<br \/>17<br \/>18<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2select<br \/>\n3 &nbsp;l_orderkey,<br \/>\n4 &nbsp;sum(l_extendedprice * (1 - l_discount)) as revenue,<br \/>\n5 &nbsp;o_orderdate,<br \/>\n6 &nbsp;o_shippriority<br \/>\n7from<br \/>\n8 &nbsp;orders,<br \/>\n9 &nbsp;lineitem,<br \/>\n10 &nbsp;customer<br \/>\n11where<br \/>\n12 &nbsp;c_mktsegment = &amp;#x27;BUILDING&amp;#x27;<br \/>\n13 &nbsp;and c_custkey = o_custkey<br \/>\n14 &nbsp;and l_orderkey = o_orderkey<br \/>\n15 &nbsp;and o_orderdate &amp;lt; &amp;#x27;1995-03-22&amp;#x27;<br \/>\n16 &nbsp;and l_shipdate &amp;gt; &amp;#x27;1995-03-22&amp;#x27;<br \/>\n17limit 10;<br \/>\n18<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u6ce8\u610f\u4e8b\u9879<\/p>\n<p><strong>Join****\u6570\u636e\u503e\u659c\u95ee\u9898<\/strong><\/p>\n<p>\u6267\u884c\u4efb\u52a1\u7684\u65f6\u5019\uff0c\u4efb\u52a1\u8fdb\u5ea6\u957f\u65f6\u95f4\u7ef4\u6301\u572899%\uff0c\u8fd9\u79cd\u73b0\u8c61\u53eb\u6570\u636e\u503e\u659c\u3002<\/p>\n<p>\u6570\u636e\u503e\u659c\u662f\u7ecf\u5e38\u5b58\u5728\u7684\uff0c\u56e0\u4e3a\u6709\u5c11\u91cf\u7684Reduce\u4efb\u52a1\u5206\u914d\u5230\u7684\u6570\u636e\u91cf\u548c\u5176\u4ed6Reduce\u5dee\u5f02\u8fc7\u5927\uff0c\u5bfc\u81f4\u5927\u90e8\u5206Reduce\u90fd\u5df2\u5b8c\u6210\u4efb\u52a1\uff0c\u4f46\u5c11\u91cfReduce\u4efb\u52a1\u8fd8\u6ca1\u5b8c\u6210\u7684\u60c5\u51b5\u3002<\/p>\n<p>\u89e3\u51b3\u6570\u636e\u503e\u659c\u7684\u95ee\u9898\uff0c\u53ef\u901a\u8fc7\u8bbe\u7f6e<em><strong>set hive.optimize.skewjoin=true<\/strong><\/em>\u5e76\u8c03\u6574hive.skewjoin.key\u7684\u5927\u5c0f\u3002hive.skewjoin.key\u662f\u6307Reduce\u7aef\u63a5\u6536\u5230\u591a\u5c11\u4e2akey\u5373\u8ba4\u4e3a\u6570\u636e\u662f\u503e\u659c\u7684\uff0c\u5e76\u81ea\u52a8\u5206\u53d1\u5230\u591a\u4e2aReduce\u3002<\/p>\n<h3>12.5.3\u00a0Group By\u4f18\u5316<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u4f18\u5316Group by\u8bed\u53e5\uff0c\u53ef\u63d0\u5347\u547d\u4ee4\u6267\u884c\u901f\u5ea6\u548c\u67e5\u8be2\u901f\u5ea6\u3002<\/p>\n<p>Group by\u7684\u65f6\u5019\uff0c Map\u7aef\u4f1a\u5148\u8fdb\u884c\u5206\u7ec4\uff0c \u5206\u7ec4\u5b8c\u540e\u5206\u53d1\u5230Reduce\u7aef\uff0c Reduce\u7aef\u518d\u8fdb\u884c\u5206\u7ec4\u3002\u53ef\u91c7\u7528Map\u7aef\u805a\u5408\u7684\u65b9\u5f0f\u6765\u8fdb\u884cGroup by\u4f18\u5316\uff0c\u5f00\u542fMap\u7aef\u521d\u6b65\u805a\u5408\uff0c\u51cf\u5c11Map\u7684\u8f93\u51fa\u6570\u636e\u91cf\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u5728Hive\u5ba2\u6237\u7aef\u8fdb\u884c\u5982\u4e0b\u8bbe\u7f6e\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2set hive.map.aggr=true<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u6ce8\u610f\u4e8b\u9879<\/p>\n<p><strong>Group By****\u6570\u636e\u503e\u659c<\/strong><\/p>\n<p>Group By\u4e5f\u540c\u6837\u5b58\u5728\u6570\u636e\u503e\u659c\u7684\u95ee\u9898\uff0c\u8bbe\u7f6ehive.groupby.skewindata\u4e3atrue\uff0c\u751f\u6210\u7684\u67e5\u8be2\u8ba1\u5212\u4f1a\u6709\u4e24\u4e2aMapReduce Job\uff0c\u7b2c\u4e00\u4e2aJob\u7684Map\u8f93\u51fa\u7ed3\u679c\u4f1a\u968f\u673a\u7684\u5206\u5e03\u5230Reduce\u4e2d\uff0c\u6bcf\u4e2aReduce\u505a\u805a\u5408\u64cd\u4f5c\uff0c\u5e76\u8f93\u51fa\u7ed3\u679c\uff0c\u8fd9\u6837\u7684\u5904\u7406\u4f1a\u4f7f\u76f8\u540c\u7684Group By Key\u53ef\u80fd\u88ab\u5206\u53d1\u5230\u4e0d\u540c\u7684Reduce\u4e2d\uff0c\u4ece\u800c\u8fbe\u5230\u8d1f\u8f7d\u5747\u8861\uff0c\u7b2c\u4e8c\u4e2aJob\u518d\u6839\u636e\u9884\u5904\u7406\u7684\u7ed3\u679c\u6309\u7167Group By Key\u5206\u53d1\u5230Reduce\u4e2d\u5b8c\u6210\u6700\u7ec8\u7684\u805a\u5408\u64cd\u4f5c\u3002<\/p>\n<p><strong>Count Distinct****\u805a\u5408\u95ee\u9898<\/strong><\/p>\n<p>\u5f53\u4f7f\u7528\u805a\u5408\u51fd\u6570count distinct\u5b8c\u6210\u53bb\u91cd\u8ba1\u6570\u65f6\uff0c\u5904\u7406\u503c\u4e3a\u7a7a\u7684\u60c5\u51b5\u4f1a\u4f7fReduce\u4ea7\u751f\u5f88\u4e25\u91cd\u7684\u6570\u636e\u503e\u659c\uff0c\u53ef\u4ee5\u5c06\u7a7a\u503c\u5355\u72ec\u5904\u7406\uff0c\u5982\u679c\u662f\u8ba1\u7b97count distinct\uff0c\u53ef\u4ee5\u901a\u8fc7where\u5b57\u53e5\u5c06\u8be5\u503c\u6392\u9664\u6389\uff0c\u5e76\u5728\u6700\u540e\u7684count distinct\u7ed3\u679c\u4e2d\u52a01\u3002\u5982\u679c\u8fd8\u6709\u5176\u4ed6\u8ba1\u7b97\uff0c\u53ef\u4ee5\u5148\u5c06\u503c\u4e3a\u7a7a\u7684\u8bb0\u5f55\u5355\u72ec\u5904\u7406\uff0c\u518d\u548c\u5176\u4ed6\u8ba1\u7b97\u7ed3\u679c\u5408\u5e76\u3002<\/p>\n<h3>12.5.4\u00a0\u6570\u636e\u5b58\u50a8\u4f18\u5316<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u201cORC\u201d\u662f\u4e00\u79cd\u9ad8\u6548\u7684\u5217\u5b58\u50a8\u683c\u5f0f\uff0c\u5728\u538b\u7f29\u6bd4\u548c\u8bfb\u53d6\u6548\u7387\u4e0a\u4f18\u4e8e\u5176\u4ed6\u6587\u4ef6\u683c\u5f0f\u3002<\/p>\n<p>\u5efa\u8bae\u4f7f\u7528\u201cORC\u201d\u4f5c\u4e3aHive\u8868\u9ed8\u8ba4\u7684\u5b58\u50a8\u683c\u5f0f\u3002<\/p>\n<p>\u524d\u63d0\u6761\u4ef6<\/p>\n<p>\u5df2\u767b\u5f55Hive\u5ba2\u6237\u7aef\uff0c\u5177\u4f53\u64cd\u4f5c\u8bf7\u53c2\u89c1\u300a\u7ba1\u7406\u5458\u6307\u5357\u300b\u7684\u201c\u4f7f\u7528Hive\u5ba2\u6237\u7aef\u201d\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ul>\n<li>\n<p>\u63a8\u8350\uff1a\u4f7f\u7528\u201cSNAPPY\u201d\u538b\u7f29\uff0c\u9002\u7528\u4e8e\u538b\u7f29\u6bd4\u548c\u8bfb\u53d6\u6548\u7387\u8981\u6c42\u5747\u8861\u573a\u666f\u3002<\/p>\n<\/li>\n<\/ul>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2Create table xx stored as orc tblproperties (&amp;quot;orc.compress&amp;quot;=&amp;quot;SNAPPY&amp;quot;)<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<ul>\n<li>\n<p>\u53ef\u7528\uff1a\u4f7f\u7528\u201cZLIB\u201d\u538b\u7f29\uff0c\u9002\u7528\u4e8e\u538b\u7f29\u6bd4\u8981\u6c42\u8f83\u9ad8\u573a\u666f\u3002<\/p>\n<\/li>\n<\/ul>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2Create table xx stored as orc tblproperties (&amp;quot;orc.compress&amp;quot;=&amp;quot;ZLIB&amp;quot;)<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>xx\u4e3a\u5177\u4f53\u4f7f\u7528\u7684Hive\u8868\u540d\u3002<\/p>\n<h3>12.5.5\u00a0SQL\u4f18\u5316<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5728Hive\u4e0a\u6267\u884cSQL\u8bed\u53e5\u67e5\u8be2\u65f6\uff0c\u5982\u679c\u8bed\u53e5\u4e2d\u5b58\u5728\u201c(a&amp;b) or (a&amp;c)\u201d\u903b\u8f91\u65f6\uff0c\u5efa\u8bae\u5c06\u903b\u8f91\u6539\u4e3a\u201ca &amp; (b or c)\u201d\u3002<\/p>\n<p>\u6837\u4f8b<\/p>\n<p>\u5047\u8bbe\u6761\u4ef6a\u4e3a\u201cp_partkey = l_partkey\u201d\uff0c\u4f18\u5316\u524d\u6837\u4f8b\u5982\u4e0b\u6240\u793a\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;height:300px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/>6<br \/>7<br \/>8<br \/>9<br \/>10<br \/>11<br \/>12<br \/>13<br \/>14<br \/>15<br \/>16<br \/>17<br \/>18<br \/>19<br \/>20<br \/>21<br \/>22<br \/>23<br \/>24<br \/>25<br \/>26<br \/>27<br \/>28<br \/>29<br \/>30<br \/>31<br \/>32<br \/>33<br \/>34<br \/>35<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2select<br \/>\n3 &nbsp; &nbsp; &nbsp; &nbsp;sum(l_extendedprice* (1 - l_discount)) as revenue<br \/>\n4from<br \/>\n5 &nbsp; &nbsp; &nbsp; &nbsp;lineitem,<br \/>\n6 &nbsp; &nbsp; &nbsp; &nbsp;part<br \/>\n7where &nbsp; <br \/>\n8 &nbsp; &nbsp; &nbsp; &nbsp;(<br \/>\n9 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;p_partkey = l_partkey <br \/>\n10 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_brand = &amp;#x27;Brand#32&amp;#x27;<br \/>\n11 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_container in (&amp;#x27;SM CASE&amp;#x27;, &amp;#x27;SM BOX&amp;#x27;, &amp;#x27;SM PACK&amp;#x27;, &amp;#x27;SM PKG&amp;#x27;)<br \/>\n12 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_quantity &amp;gt;= 7 and l_quantity &amp;lt;= 7 + 10<br \/>\n13 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_size between 1 and 5<br \/>\n14 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_shipmode in (&amp;#x27;AIR&amp;#x27;, &amp;#x27;AIR REG&amp;#x27;)<br \/>\n15 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_shipinstruct = &amp;#x27;DELIVER IN PERSON&amp;#x27;<br \/>\n16 &nbsp; &nbsp; &nbsp; &nbsp;)<br \/>\n17 &nbsp; &nbsp; &nbsp; &nbsp;or<br \/>\n18 &nbsp; &nbsp; &nbsp; &nbsp;( &nbsp; &nbsp; &nbsp; p_partkey = l_partkey <br \/>\n19 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_brand = &amp;#x27;Brand#35&amp;#x27;<br \/>\n20 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_container in (&amp;#x27;MED BAG&amp;#x27;, &amp;#x27;MED BOX&amp;#x27;, &amp;#x27;MED PKG&amp;#x27;, &amp;#x27;MED PACK&amp;#x27;)<br \/>\n21 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_quantity &amp;gt;= 15 and l_quantity &amp;lt;= 15 + 10<br \/>\n22 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_size between 1 and 10<br \/>\n23 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_shipmode in (&amp;#x27;AIR&amp;#x27;, &amp;#x27;AIR REG&amp;#x27;)<br \/>\n24 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_shipinstruct = &amp;#x27;DELIVER IN PERSON&amp;#x27;<br \/>\n25 &nbsp; &nbsp; &nbsp; &nbsp;)<br \/>\n26 &nbsp; &nbsp; &nbsp; &nbsp;or<br \/>\n27 &nbsp; &nbsp; &nbsp; &nbsp;( &nbsp; &nbsp; &nbsp; p_partkey = l_partkey <br \/>\n28 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_brand = &amp;#x27;Brand#24&amp;#x27;<br \/>\n29 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_container in (&amp;#x27;LG CASE&amp;#x27;, &amp;#x27;LG BOX&amp;#x27;, &amp;#x27;LG PACK&amp;#x27;, &amp;#x27;LG PKG&amp;#x27;)<br \/>\n30 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_quantity &amp;gt;= 26 and l_quantity &amp;lt;= 26 + 10<br \/>\n31 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_size between 1 and 15<br \/>\n32 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_shipmode in (&amp;#x27;AIR&amp;#x27;, &amp;#x27;AIR REG&amp;#x27;)<br \/>\n33 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_shipinstruct = &amp;#x27;DELIVER IN PERSON&amp;#x27;<br \/>\n34 &nbsp; &nbsp; &nbsp; &nbsp;)<br \/>\n35<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u4f18\u5316\u540e\u6837\u4f8b\u5982\u4e0b\u6240\u793a\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;height:300px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/>6<br \/>7<br \/>8<br \/>9<br \/>10<br \/>11<br \/>12<br \/>13<br \/>14<br \/>15<br \/>16<br \/>17<br \/>18<br \/>19<br \/>20<br \/>21<br \/>22<br \/>23<br \/>24<br \/>25<br \/>26<br \/>27<br \/>28<br \/>29<br \/>30<br \/>31<br \/>32<br \/>33<br \/>34<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2select<br \/>\n3 &nbsp; &nbsp; &nbsp; &nbsp;sum(l_extendedprice* (1 - l_discount)) as revenue<br \/>\n4from<br \/>\n5 &nbsp; &nbsp; &nbsp; &nbsp;lineitem,<br \/>\n6 &nbsp; &nbsp; &nbsp; &nbsp;part<br \/>\n7where &nbsp; p_partkey = l_partkey and<br \/>\n8 &nbsp; &nbsp; &nbsp; &nbsp;((<br \/>\n9 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;p_brand = &amp;#x27;Brand#32&amp;#x27;<br \/>\n10 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_container in (&amp;#x27;SM CASE&amp;#x27;, &amp;#x27;SM BOX&amp;#x27;, &amp;#x27;SM PACK&amp;#x27;, &amp;#x27;SM PKG&amp;#x27;)<br \/>\n11 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_quantity &amp;gt;= 7 and l_quantity &amp;lt;= 7 + 10<br \/>\n12 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_size between 1 and 5<br \/>\n13 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_shipmode in (&amp;#x27;AIR&amp;#x27;, &amp;#x27;AIR REG&amp;#x27;)<br \/>\n14 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_shipinstruct = &amp;#x27;DELIVER IN PERSON&amp;#x27;<br \/>\n15 &nbsp; &nbsp; &nbsp; &nbsp;)<br \/>\n16 &nbsp; &nbsp; &nbsp; &nbsp;or<br \/>\n17 &nbsp; &nbsp; &nbsp; &nbsp;(<br \/>\n18 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;p_brand = &amp;#x27;Brand#35&amp;#x27;<br \/>\n19 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_container in (&amp;#x27;MED BAG&amp;#x27;, &amp;#x27;MED BOX&amp;#x27;, &amp;#x27;MED PKG&amp;#x27;, &amp;#x27;MED PACK&amp;#x27;)<br \/>\n20 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_quantity &amp;gt;= 15 and l_quantity &amp;lt;= 15 + 10<br \/>\n21 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_size between 1 and 10<br \/>\n22 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_shipmode in (&amp;#x27;AIR&amp;#x27;, &amp;#x27;AIR REG&amp;#x27;)<br \/>\n23 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_shipinstruct = &amp;#x27;DELIVER IN PERSON&amp;#x27;<br \/>\n24 &nbsp; &nbsp; &nbsp; &nbsp;)<br \/>\n25 &nbsp; &nbsp; &nbsp; &nbsp;or<br \/>\n26 &nbsp; &nbsp; &nbsp; &nbsp;(<br \/>\n27 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;p_brand = &amp;#x27;Brand#24&amp;#x27;<br \/>\n28 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_container in (&amp;#x27;LG CASE&amp;#x27;, &amp;#x27;LG BOX&amp;#x27;, &amp;#x27;LG PACK&amp;#x27;, &amp;#x27;LG PKG&amp;#x27;)<br \/>\n29 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_quantity &amp;gt;= 26 and l_quantity &amp;lt;= 26 + 10<br \/>\n30 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and p_size between 1 and 15<br \/>\n31 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_shipmode in (&amp;#x27;AIR&amp;#x27;, &amp;#x27;AIR REG&amp;#x27;)<br \/>\n32 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;and l_shipinstruct = &amp;#x27;DELIVER IN PERSON&amp;#x27;<br \/>\n33 &nbsp; &nbsp; &nbsp; &nbsp;))<br \/>\n34<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<h3>12.5.6\u00a0\u4f7f\u7528Hive CBO\u4f18\u5316\u67e5\u8be2<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5728Hive\u4e2d\u6267\u884c\u591a\u8868Join\u65f6\uff0cHive\u652f\u6301\u5f00\u542fCBO\uff08Cost Based Optimization\uff09\uff0c\u7cfb\u7edf\u4f1a\u81ea\u52a8\u6839\u636e\u8868\u7684\u7edf\u8ba1\u4fe1\u606f\uff0c\u4f8b\u5982\u6570\u636e\u91cf\u3001\u6587\u4ef6\u6570\u7b49\uff0c\u9009\u51fa\u6700\u4f18\u8ba1\u5212\u63d0\u9ad8\u591a\u8868Join\u7684\u6548\u7387\u3002Hive\u9700\u8981\u5148\u6536\u96c6\u8868\u7684\u7edf\u8ba1\u4fe1\u606f\u540e\u624d\u80fd\u4f7fCBO\u6b63\u786e\u7684\u4f18\u5316\u3002<\/p>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<ul>\n<li>CBO\u4f18\u5316\u5668\u4f1a\u57fa\u4e8e\u7edf\u8ba1\u4fe1\u606f\u548c\u67e5\u8be2\u6761\u4ef6\uff0c\u5c3d\u53ef\u80fd\u5730\u4f7fjoin\u987a\u5e8f\u8fbe\u5230\u6700\u4f18\u3002\u4f46\u662f\u4e5f\u53ef\u80fd\u5b58\u5728\u7279\u6b8a\u60c5\u51b5\u5bfc\u81f4join\u987a\u5e8f\u8c03\u6574\u4e0d\u51c6\u786e\u3002\u4f8b\u5982\u6570\u636e\u5b58\u5728\u503e\u659c\uff0c\u4ee5\u53ca\u67e5\u8be2\u6761\u4ef6\u503c\u5728\u8868\u4e2d\u4e0d\u5b58\u5728\u7b49\u573a\u666f\uff0c\u53ef\u80fd\u8c03\u6574\u51fa\u975e\u4f18\u5316\u7684join\u987a\u5e8f\u3002<\/li>\n<li>\u5f00\u542f\u5217\u7edf\u8ba1\u4fe1\u606f\u81ea\u52a8\u6536\u96c6\u65f6\uff0c\u9700\u8981\u5728reduce\u4fa7\u505a\u805a\u5408\u7edf\u8ba1\u3002\u5bf9\u4e8e\u6ca1\u6709reduce\u9636\u6bb5\u7684insert\u4efb\u52a1\uff0c\u5c06\u4f1a\u591a\u51fareduce\u9636\u6bb5\uff0c\u7528\u4e8e\u6536\u96c6\u7edf\u8ba1\u4fe1\u606f\u3002<\/li>\n<\/ul>\n<p>\u524d\u63d0\u6761\u4ef6<\/p>\n<p>\u5df2\u767b\u5f55Hive\u5ba2\u6237\u7aef\uff0c\u5177\u4f53\u64cd\u4f5c\u8bf7\u53c2\u89c1\u300a\u7ba1\u7406\u5458\u6307\u5357\u300b\u7684\u201c\u4f7f\u7528Hive\u5ba2\u6237\u7aef\u201d\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ol>\n<li>\u5728Manager\u754c\u9762Hive\u7ec4\u4ef6\u7684\u670d\u52a1\u914d\u7f6e\u4e2d\u641c\u7d22\u201chive.cbo.enable\u201d\u53c2\u6570\uff0c\u9009\u4e2d\u201ctrue\u201d\u6c38\u4e45\u5f00\u542f\u529f\u80fd\u6216\u8005\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u4e34\u65f6\u5f00\u542f\u529f\u80fd\uff1a<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p><strong>set hive.cbo.enable=true;<\/strong><\/p>\n<p>\u00a0<\/p>\n<ol>\n<li>\u624b\u52a8\u6536\u96c6Hive\u8868\u5df2\u6709\u6570\u636e\u7684\u7edf\u8ba1\u4fe1\u606f\u3002<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p>\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u53ef\u4ee5\u624b\u52a8\u6536\u96c6\u7edf\u8ba1\u4fe1\u606f\u3002\u4ec5\u652f\u6301\u7edf\u8ba1\u4e00\u5f20\u8868\uff0c\u5982\u679c\u9700\u8981\u7edf\u8ba1\u4e0d\u540c\u7684\u8868\u9700\u91cd\u590d\u6267\u884c\u3002<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/>6<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2ANALYZE TABLE [db_name.]tablename [PARTITION(partcol1[=val1], partcol2[=val2], ...)]<br \/>\n3COMPUTE STATISTICS<br \/>\n4[FOR COLUMNS]<br \/>\n5[NOSCAN];<br \/>\n6<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/><\/div><\/td><td><div class=\"text codecolorer\">1 \u00a0\u8bf4\u660e\uff1a <br \/>\n2<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<ul>\n<li>\u6307\u5b9aFOR COLUMNS\u65f6\uff0c\u6536\u96c6\u5217\u7ea7\u522b\u7684\u7edf\u8ba1\u4fe1\u606f\u3002<!-- -->\n<ul>\n<li>\u6307\u5b9aNOSCAN\u65f6\uff0c\u5c06\u53ea\u7edf\u8ba1\u6587\u4ef6\u5927\u5c0f\u548c\u4e2a\u6570\uff0c\u4e0d\u626b\u63cf\u5177\u4f53\u6587\u4ef6\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u4f8b\u5982\uff1a<\/p>\n<p><strong>analyze table table_name compute statistics;<\/strong><\/p>\n<p><strong>analyze table table_name compute statistics for columns;<\/strong><\/p>\n<p>\u00a0<\/p>\n<ul>\n<li>\u914d\u7f6eHive\u81ea\u52a8\u6536\u96c6\u7edf\u8ba1\u4fe1\u606f\u3002\u5f00\u542f\u914d\u7f6e\u540e\uff0c\u6267\u884c<strong>insert overwrite\/into<\/strong>\u547d\u4ee4\u63d2\u5165\u6570\u636e\u65f6\u624d\u81ea\u52a8\u7edf\u8ba1\u65b0\u6570\u636e\u7684\u4fe1\u606f\u3002<\/li>\n<\/ul>\n<p>\u00a0<\/p>\n<ul>\n<li>\u5728Hive\u5ba2\u6237\u7aef\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u4e34\u65f6\u5f00\u542f\u6536\u96c6\uff1a<\/li>\n<\/ul>\n<p><strong>set hive.stats.autogather = true;<\/strong>\u00a0\u5f00\u542f\u8868\/\u5206\u533a\u7ea7\u522b\u7684\u7edf\u8ba1\u4fe1\u606f\u81ea\u52a8\u6536\u96c6\u3002<\/p>\n<p>**set hive.stats.column.autogather = true;\u00a0**\u5f00\u542f\u5217\u7ea7\u522b\u7684\u7edf\u8ba1\u4fe1\u606f\u81ea\u52a8\u6536\u96c6\u3002<br \/>\n\u00a0\u8bf4\u660e\uff1a<\/p>\n<ul>\n<li>\n<p>\u5217\u7ea7\u522b\u7edf\u8ba1\u4fe1\u606f\u7684\u6536\u96c6\u4e0d\u652f\u6301\u590d\u6742\u7684\u6570\u636e\u7c7b\u578b\uff0c\u4f8b\u5982Map\uff0cStruct\u7b49\u3002<br \/>\n* \u8868\u7ea7\u522b\u7edf\u8ba1\u4fe1\u606f\u7684\u81ea\u52a8\u6536\u96c6\u4e0d\u652f\u6301Hive on HBase\u8868\u3002<\/p>\n<\/li>\n<li>\n<p>\u5728Manager\u754c\u9762Hive\u7684\u670d\u52a1\u914d\u7f6e\u4e2d\uff0c\u641c\u7d22\u53c2\u6570\u201chive.stats.autogather\u201d\u548c\u201chive.stats.column.autogather\u201d\uff0c\u9009\u4e2d\u201ctrue\u201d\u6c38\u4e45\u5f00\u542f\u6536\u96c6\u529f\u80fd\u3002<\/p>\n<\/li>\n<\/ul>\n<p>\u00a0<\/p>\n<ul>\n<li>\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u53ef\u4ee5\u67e5\u770b\u7edf\u8ba1\u4fe1\u606f\u3002<\/li>\n<\/ul>\n<p>\u00a0<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2DESCRIBE FORMATTED table_name[.column_name] PARTITION partition_spec;<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u4f8b\u5982\uff1a<\/p>\n<p><strong>desc formatted table_name;<\/strong><\/p>\n<p><strong>desc formatted table_name.id;<\/strong><\/p>\n<p><strong>desc formatted table_name.id partition(time=&#x27;2016-05-27&#x27;);<\/strong><br \/>\n\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u5206\u533a\u8868\u4ec5\u652f\u6301\u5206\u533a\u7ea7\u522b\u7684\u7edf\u8ba1\u4fe1\u606f\u6536\u96c6\uff0c\u56e0\u6b64\u5206\u533a\u8868\u9700\u8981\u6307\u5b9a\u5206\u533a\u6765\u67e5\u8be2\u7edf\u8ba1\u4fe1\u606f\u3002<\/p>\n<p>\u00a0<\/p>\n<h2>12.6\u00a0Kafka<\/h2>\n<h3>12.6.1\u00a0Kafka\u6027\u80fd\u8c03\u4f18<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u901a\u8fc7\u8c03\u6574Kafka\u670d\u52a1\u7aef\u53c2\u6570\uff0c\u53ef\u4ee5\u63d0\u5347\u7279\u5b9a\u4e1a\u52a1\u573a\u666f\u4e0bKafka\u7684\u5904\u7406\u80fd\u529b\u3002<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; Kafka &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<\/p>\n<p>\u53c2\u6570\u8c03\u4f18<\/p>\n<p>\u886812-15\u00a0\u8c03\u4f18\u53c2\u6570<\/p>\n<p>num.recovery.threads.per.data.dir<br \/>\n10<br \/>\n\u5728Kafka\u542f\u52a8\u8fc7\u7a0b\u4e2d\uff0c\u6570\u636e\u91cf\u8f83\u5927\u60c5\u51b5\u4e0b\uff0c\u53ef\u8c03\u5927\u6b64\u53c2\u6570\uff0c\u53ef\u4ee5\u63d0\u5347\u542f\u52a8\u901f\u5ea6\u3002<br \/>\nbackground.threads<br \/>\n10<br \/>\nBroker\u540e\u53f0\u4efb\u52a1\u5904\u7406\u7684\u7ebf\u7a0b\u6570\u76ee\u3002\u6570\u636e\u91cf\u8f83\u5927\u7684\u60c5\u51b5\u4e0b\uff0c\u53ef\u9002\u5f53\u8c03\u5927\u6b64\u53c2\u6570\uff0c\u4ee5\u63d0\u5347Broker\u5904\u7406\u80fd\u529b\u3002<br \/>\nnum.replica.fetchers<br \/>\n1<br \/>\n\u526f\u672c\u5411Leader\u8bf7\u6c42\u540c\u6b65\u6570\u636e\u7684\u7ebf\u7a0b\u6570\uff0c\u589e\u5927\u8fd9\u4e2a\u6570\u503c\u4f1a\u589e\u52a0\u526f\u672c\u7684I\/O\u5e76\u53d1\u5ea6\u3002<br \/>\nnum.io.threads<br \/>\n8<br \/>\nBroker\u7528\u6765\u5904\u7406\u78c1\u76d8I\/O\u7684\u7ebf\u7a0b\u6570\u76ee\uff0c\u8fd9\u4e2a\u7ebf\u7a0b\u6570\u76ee\u5efa\u8bae\u81f3\u5c11\u7b49\u4e8e\u786c\u76d8\u7684\u4e2a\u6570\u3002<br \/>\nKAFKA_HEAP_OPTS<br \/>\n-Xmx6G<br \/>\nKafka JVM\u5806\u5185\u5b58\u8bbe\u7f6e\u3002\u5f53Broker\u4e0a\u6570\u636e\u91cf\u8f83\u5927\u65f6\uff0c\u5e94\u9002\u5f53\u8c03\u6574\u5806\u5185\u5b58\u5927\u5c0f\u3002<\/p>\n<h2>12.7\u00a0MapReduce<\/h2>\n<h3>12.7.1\u00a0\u591aCPU\u5185\u6838\u4e0b\u7684\u8c03\u4f18\u914d\u7f6e<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5f53CPU\u5185\u6838\u6570\u5f88\u591a\u65f6\uff0c\u5982CPU\u5185\u6838\u4e3a\u78c1\u76d8\u6570\u76843\u500d\u65f6\u7684\u8c03\u4f18\u914d\u7f6e\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u4ee5\u4e0b\u53c2\u6570\u6709\u5982\u4e0b\u4e24\u4e2a\u914d\u7f6e\u5165\u53e3\uff1a<\/p>\n<ul>\n<li>\u670d\u52a1\u5668\u7aef\u914d\u7f6e<\/li>\n<\/ul>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; YARN &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<\/p>\n<ul>\n<li>\n<p>\u5ba2\u6237\u7aef\u914d\u7f6e\u76f4\u63a5\u5728\u5ba2\u6237\u7aef\u4e2d\u4fee\u6539\u76f8\u5e94\u7684\u914d\u7f6e\u6587\u4ef6\u3002\u00a0\u8bf4\u660e\uff1a<\/p>\n<\/li>\n<li>\n<p>HDFS\u5ba2\u6237\u7aef\u914d\u7f6e\u6587\u4ef6\u8def\u5f84\uff1a\u5ba2\u6237\u7aef\u5b89\u88c5\u76ee\u5f55\/HDFS\/hadoop\/etc\/hadoop\/hdfs-site.xml\u3002<\/p>\n<\/li>\n<li>\n<p>Yarn\u5ba2\u6237\u7aef\u914d\u7f6e\u6587\u4ef6\u8def\u5f84\uff1a\u5ba2\u6237\u7aef\u5b89\u88c5\u76ee\u5f55\/HDFS\/hadoop\/etc\/hadoop\/yarn-site.xml\u3002<\/p>\n<\/li>\n<li>\n<p>MapReduce\u5ba2\u6237\u7aef\u914d\u7f6e\u6587\u4ef6\u8def\u5f84\uff1a\u5ba2\u6237\u7aef\u5b89\u88c5\u76ee\u5f55\/HDFS\/hadoop\/etc\/hadoop\/mapred-site.xml\u3002<\/p>\n<\/li>\n<\/ul>\n<p>\u886812-16\u00a0\u591aCPU\u5185\u6838\u8bbe\u7f6e<\/p>\n<h2>\u8282\u70b9\u5bb9\u5668\u69fd\u4f4d\u6570 \u00a0<br \/>\n\u5982\u4e0b\u914d\u7f6e\u7ec4\u5408\u51b3\u5b9a\u4e86\u6bcf\u8282\u70b9\u4efb\u52a1(map\u3001reduce)\u7684\u5e76\u53d1\u6570\u3002 \u201cyarn.nodemanager.resource.memory-mb\u201d \u201cmapreduce.map.memory.mb\u201d \u201cmapreduce.reduce.memory.mb\u201d<br \/>\nyarn.nodemanager.resource.memory-mb \u8bf4\u660e\uff1a \u9700\u8981\u5728FusionInsight Manager\u7cfb\u7edf\u8fdb\u884c\u914d\u7f6e\u3002 \u00a0<br \/>\n8192<br \/>\nServer<br \/>\n\u5982\u679c\u6240\u6709\u7684\u4efb\u52a1(map\/reduce)\u9700\u8981\u8bfb\u5199\u6570\u636e\u81f3\u78c1\u76d8\uff0c\u591a\u4e2a\u8fdb\u7a0b\u5c06\u4f1a\u540c\u65f6\u8bbf\u95ee\u4e00\u4e2a\u78c1\u76d8\u3002\u8fd9\u5c06\u4f1a\u5bfc\u81f4\u78c1\u76d8\u7684IO\u6027\u80fd\u975e\u5e38\u7684\u4f4e\u4e0b\u3002\u4e3a\u4e86\u6539\u5584\u78c1\u76d8\u7684\u6027\u80fd\uff0c\u8bf7\u786e\u4fdd\u5ba2\u6237\u7aef\u5e76\u53d1\u8bbf\u95ee\u78c1\u76d8\u7684\u6570\u4e0d\u5927\u4e8e3\u3002<br \/>\n\u6700\u5927\u5e76\u53d1\u7684container\u6570\u91cf\u5e94\u8be5\u4e3a[2.5 * Hadoop\u4e2d\u78c1\u76d8\u914d\u7f6e\u6570 ]\u3002<br \/>\nmapreduce.map.memory.mb \u8bf4\u660e\uff1a \u9700\u8981\u5728\u5ba2\u6237\u7aef\u8fdb\u884c\u914d\u7f6e\uff0c\u914d\u7f6e\u6587\u4ef6\u8def\u5f84\uff1a\u5ba2\u6237\u7aef\u5b89\u88c5\u76ee\u5f55\/HDFS\/hadoop\/etc\/hadoop\/mapred-site.xml\u3002<br \/>\n4096<br \/>\nClient<br \/>\nmapreduce.reduce.memory.mb \u8bf4\u660e\uff1a \u9700\u8981\u5728\u5ba2\u6237\u7aef\u8fdb\u884c\u914d\u7f6e\uff0c\u914d\u7f6e\u6587\u4ef6\u8def\u5f84\uff1a\u5ba2\u6237\u7aef\u5b89\u88c5\u76ee\u5f55\/HDFS\/hadoop\/etc\/hadoop\/mapred-site.xml\u3002<br \/>\n4096<br \/>\nClient<br \/>\nMap\u8f93\u51fa\u4e0e\u538b\u7f29<br \/>\nMap\u4efb\u52a1\u6240\u4ea7\u751f\u7684\u8f93\u51fa\u53ef\u4ee5\u5728\u5199\u5165\u78c1\u76d8\u4e4b\u524d\u88ab\u538b\u7f29\uff0c\u8fd9\u6837\u53ef\u4ee5\u8282\u7ea6\u78c1\u76d8\u7a7a\u95f4\u5e76\u5f97\u5230\u66f4\u5feb\u7684\u5199\u76d8\u901f\u5ea6\uff0c\u540c\u65f6\u53ef\u4ee5\u51cf\u5c11\u81f3Reducer\u7684\u6570\u636e\u4f20\u8f93\u91cf\u3002\u9700\u8981\u5728\u5ba2\u6237\u7aef\u8fdb\u884c\u914d\u7f6e\u3002 mapreduce.map.output.compress\u6307\u5b9a\u4e86Map\u4efb\u52a1\u8f93\u51fa\u7ed3\u679c\u53ef\u4ee5\u5728\u7f51\u7edc\u4f20\u8f93\u524d\u88ab\u538b\u7f29\u3002\u8fd9\u662f\u4e00\u4e2aper-job\u7684\u914d\u7f6e\u3002 mapreduce.map.output.compress.codec\u6307\u5b9a\u7528\u4e8e\u538b\u7f29\u7684\u7f16\u89e3\u7801\u5668\u3002<br \/>\nmapreduce.map.output.compress \u8bf4\u660e\uff1a \u9700\u8981\u5728\u5ba2\u6237\u7aef\u8fdb\u884c\u914d\u7f6e\uff0c\u914d\u7f6e\u6587\u4ef6\u8def\u5f84\uff1a\u5ba2\u6237\u7aef\u5b89\u88c5\u76ee\u5f55\/HDFS\/hadoop\/etc\/hadoop\/mapred-site.xml\u3002<br \/>\ntrue<br \/>\nClient<br \/>\n\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u78c1\u76d8\u7684IO\u662f\u4e3b\u8981\u74f6\u9888\u3002\u6240\u4ee5\u53ef\u4ee5\u9009\u62e9\u4e00\u79cd\u538b\u7f29\u7387\u975e\u5e38\u9ad8\u7684\u538b\u7f29\u7b97\u6cd5\u3002<br \/>\n\u7f16\u89e3\u7801\u5668\u53ef\u914d\u7f6e\u4e3aSnappy\uff0cBenchmark\u6d4b\u8bd5\u7ed3\u679c\u663e\u793aSnappy\u662f\u975e\u5e38\u5e73\u8861\u4ee5\u53ca\u9ad8\u6548\u7684\u7f16\u7801\u5668\u3002<br \/>\nmapreduce.map.output.compress.codec \u8bf4\u660e\uff1a \u9700\u8981\u5728\u5ba2\u6237\u7aef\u8fdb\u884c\u914d\u7f6e\uff0c\u914d\u7f6e\u6587\u4ef6\u8def\u5f84\uff1a\u5ba2\u6237\u7aef\u5b89\u88c5\u76ee\u5f55\/HDFS\/hadoop\/etc\/hadoop\/mapred-site.xml\u3002<br \/>\norg.apache.hadoop.io.compress.SnappyCodec<br \/>\nClient<br \/>\nSpills<br \/>\nmapreduce.map.sort.spill.percent<br \/>\nmapreduce.map.sort.spill.percent \u8bf4\u660e\uff1a \u9700\u8981\u5728\u5ba2\u6237\u7aef\u8fdb\u884c\u914d\u7f6e\uff0c\u914d\u7f6e\u6587\u4ef6\u8def\u5f84\uff1a\u5ba2\u6237\u7aef\u5b89\u88c5\u76ee\u5f55\/HDFS\/hadoop\/etc\/hadoop\/mapred-site.xml\u3002<br \/>\n0.8<br \/>\nClient<br \/>\n\u78c1\u76d8IO\u662f\u4e3b\u8981\u74f6\u9888\uff0c\u5408\u7406\u914d\u7f6e\u201cmapreduce.task.io.sort.mb\u201d\u53ef\u4ee5\u4f7f\u6ea2\u51fa\u81f3\u78c1\u76d8\u7684\u5185\u5bb9\u6700\u5c0f\u5316\u3002<\/h2>\n<p>\u6570\u636e\u5305\u5927\u5c0f<br \/>\n\u5f53HDFS\u5ba2\u6237\u7aef\u5199\u6570\u636e\u81f3\u6570\u636e\u8282\u70b9\u65f6\uff0c\u6570\u636e\u4f1a\u88ab\u7d2f\u79ef\uff0c\u76f4\u5230\u5f62\u6210\u4e00\u4e2a\u5305\u3002\u7136\u540e\u8fd9\u4e2a\u6570\u636e\u5305\u4f1a\u901a\u8fc7\u7f51\u7edc\u4f20\u8f93\u3002dfs.client-write-packet-size\u914d\u7f6e\u9879\u53ef\u4ee5\u6307\u5b9a\u8be5\u6570\u636e\u5305\u7684\u5927\u5c0f\u3002\u8fd9\u4e2a\u53ef\u4ee5\u901a\u8fc7\u6bcf\u4e2ajob\u8fdb\u884c\u6307\u5b9a\u3002<br \/>\ndfs.client-write-packet-size \u8bf4\u660e\uff1a \u9700\u8981\u5728\u5ba2\u6237\u7aef\u8fdb\u884c\u914d\u7f6e\uff0c\u914d\u7f6e\u6587\u4ef6\u8def\u5f84\uff1a\u5ba2\u6237\u7aef\u5b89\u88c5\u76ee\u5f55\/HDFS\/hadoop\/etc\/hadoop\/hdfs-site.xml\u3002<br \/>\n262144<br \/>\nClient<br \/>\n\u6570\u636e\u8282\u70b9\u4eceHDFS\u5ba2\u6237\u7aef\u63a5\u6536\u6570\u636e\u5305\uff0c\u7136\u540e\u5c06\u6570\u636e\u5305\u91cc\u7684\u6570\u636e\u5355\u7ebf\u7a0b\u5199\u5165\u78c1\u76d8\u3002\u5f53\u78c1\u76d8\u5904\u4e8e\u5e76\u53d1\u5199\u5165\u72b6\u6001\u65f6\uff0c\u589e\u52a0\u6570\u636e\u5305\u7684\u5927\u5c0f\u53ef\u4ee5\u51cf\u5c11\u78c1\u76d8\u5bfb\u9053\u65f6\u95f4\uff0c\u4ece\u800c\u63d0\u5347IO\u6027\u80fd\u3002<br \/>\ndfs.client-write-packet-size = 262144<\/p>\n<h3>12.7.2\u00a0\u786e\u5b9aJob\u57fa\u7ebf<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u786e\u5b9aJob\u57fa\u7ebf\u662f\u8c03\u4f18\u7684\u57fa\u7840\uff0c\u4e00\u5207\u8c03\u4f18\u9879\u6548\u679c\u7684\u68c0\u67e5\uff0c\u90fd\u662f\u901a\u8fc7\u548c\u57fa\u7ebf\u6570\u636e\u505a\u5bf9\u6bd4\u6765\u83b7\u5f97\u3002<\/p>\n<p>Job\u57fa\u7ebf\u7684\u786e\u5b9a\u6709\u5982\u4e0b\u4e09\u4e2a\u539f\u5219\uff1a<\/p>\n<ul>\n<li>\u5145\u5206\u5229\u7528\u96c6\u7fa4\u8d44\u6e90<\/li>\n<li>reduce\u9636\u6bb5\u5c3d\u91cf\u653e\u5728\u4e00\u8f6e<\/li>\n<li>\u6bcf\u4e2atask\u7684\u6267\u884c\u65f6\u95f4\u8981\u5408\u7406<\/li>\n<\/ul>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ul>\n<li><strong>\u539f\u5219\u4e00\uff1a\u5145\u5206\u5229\u7528\u96c6\u7fa4\u8d44\u6e90\u3002<\/strong><\/li>\n<\/ul>\n<p>Job\u8fd0\u884c\u65f6\uff0c\u4f1a\u8ba9\u6240\u6709\u7684\u8282\u70b9\u90fd\u6709\u4efb\u52a1\u5904\u7406\uff0c\u4e14\u5904\u4e8e\u7e41\u5fd9\u72b6\u6001\uff0c\u8fd9\u6837\u624d\u80fd\u4fdd\u8bc1\u8d44\u6e90\u5145\u5206\u5229\u7528\uff0c\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u8fbe\u5230\u6700\u5927\u3002\u53ef\u4ee5\u901a\u8fc7\u8c03\u6574\u5904\u7406\u7684\u6570\u636e\u91cf\u5927\u5c0f\uff0c\u4ee5\u53ca\u8c03\u6574map\u548creduce\u4e2a\u6570\u6765\u5b9e\u73b0\u3002<\/p>\n<p>Reduce\u4e2a\u6570\u7684\u63a7\u5236\u4f7f\u7528\u201cmapreduce.job.reduces\u201d\u3002<\/p>\n<p>Map\u4e2a\u6570\u53d6\u51b3\u4e8e\u4f7f\u7528\u4e86\u54ea\u79cdInputFormat\uff0c\u4ee5\u53ca\u5f85\u5904\u7406\u7684\u6570\u636e\u6587\u4ef6\u662f\u5426\u53ef\u5206\u5272\u3002\u9ed8\u8ba4\u7684TextFileInputFormat\u5c06\u6839\u636eblock\u7684\u4e2a\u6570\u6765\u5206\u914dmap\u6570(\u4e00\u4e2ablock\u4e00\u4e2amap)\u3002\u901a\u8fc7\u5982\u4e0b\u914d\u7f6e\u53c2\u6570\u8fdb\u884c\u8c03\u6574\u3002<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; YARN &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<\/p>\n<h2>mapreduce.input.fileinputformat.split.maxsize<br \/>\n\u53ef\u4ee5\u8bbe\u7f6e\u6570\u636e\u5206\u7247\u7684\u6570\u636e\u6700\u5927\u503c\u3002 \u7531\u7528\u6237\u5b9a\u4e49\u7684\u5206\u7247\u5927\u5c0f\u7684\u8bbe\u7f6e\u53ca\u6bcf\u4e2a\u6587\u4ef6block\u5927\u5c0f\u7684\u8bbe\u7f6e\uff0c\u53ef\u4ee5\u8ba1\u7b97\u5206\u7247\u7684\u5927\u5c0f\u3002\u8ba1\u7b97\u516c\u5f0f\u5982\u4e0b\uff1a<br \/>\nsplitSize = Math.max(minSize, Math.min(maxSize, blockSize)) \u5982\u679cmaxSize\u8bbe\u7f6e\u5927\u4e8eblockSize\uff0c\u90a3\u4e48\u6bcf\u4e2ablock\u5c31\u662f\u4e00\u4e2a\u5206\u7247\uff0c\u5426\u5219\u5c31\u4f1a\u5c06\u4e00\u4e2ablock\u6587\u4ef6\u5206\u9694\u4e3a\u591a\u4e2a\u5206\u7247\uff0c\u5982\u679cblock\u4e2d\u5269\u4e0b\u7684\u4e00\u5c0f\u6bb5\u6570\u636e\u91cf\u5c0f\u4e8esplitSize\uff0c\u8fd8\u662f\u8ba4\u4e3a\u5b83\u662f\u72ec\u7acb\u7684\u5206\u7247\u3002<\/h2>\n<p>mapreduce.input.fileinputformat.split.minsize<br \/>\n\u53ef\u4ee5\u8bbe\u7f6e\u6570\u636e\u5206\u7247\u7684\u6570\u636e\u6700\u5c0f\u503c\u3002<br \/>\n0<\/p>\n<ul>\n<li>\n<p>**\u539f\u5219\u4e8c\uff1a\u63a7\u5236reduce\u9636\u6bb5\u5728\u4e00\u8f6e\u4e2d\u5b8c\u6210\u3002**\u907f\u514d\u4ee5\u4e0b\u4e24\u79cd\u573a\u666f\uff1a<\/p>\n<\/li>\n<li>\n<p>\u5927\u90e8\u5206\u7684reduce\u5728\u7b2c\u4e00\u8f6e\u8fd0\u884c\u5b8c\u540e\uff0c\u5269\u4e0b\u552f\u4e00\u4e00\u4e2areduce\u7ee7\u7eed\u8fd0\u884c\u3002\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u8fd9\u4e2areduce\u7684\u6267\u884c\u65f6\u95f4\u5c06\u6781\u5927\u5f71\u54cd\u8fd9\u4e2ajob\u7684\u8fd0\u884c\u65f6\u95f4\u3002\u56e0\u6b64\u9700\u8981\u5c06reduce\u4e2a\u6570\u51cf\u5c11\u3002<\/p>\n<ul>\n<li>\u6240\u6709\u7684map\u8fd0\u884c\u5b8c\u540e\uff0c\u53ea\u6709\u4e2a\u522b\u8282\u70b9\u6709reduce\u5728\u8fd0\u884c\u3002\u8fd9\u65f6\u5019\u96c6\u7fa4\u8d44\u6e90\u6ca1\u6709\u5f97\u5230\u5145\u5206\u5229\u7528\uff0c\u9700\u8981\u589e\u52a0reduce\u7684\u4e2a\u6570\u4ee5\u4fbf\u6bcf\u4e2a\u8282\u70b9\u90fd\u6709\u4efb\u52a1\u5904\u7406\u3002<\/li>\n<\/ul>\n<\/li>\n<li>\n<p><strong>\u539f\u5219\u4e09\uff1a\u6bcf\u4e2atask\u7684\u6267\u884c\u65f6\u95f4\u8981\u5408\u7406\u3002<\/strong><\/p>\n<\/li>\n<\/ul>\n<p>\u5982\u679c\u4e00\u4e2ajob\uff0c\u6bcf\u4e2amap\u6216reduce\u7684\u6267\u884c\u65f6\u95f4\u53ea\u6709\u51e0\u79d2\u949f\uff0c\u5c31\u610f\u5473\u7740\u8fd9\u4e2ajob\u7684\u5927\u90e8\u5206\u65f6\u95f4\u90fd\u6d88\u8017\u5728task\u7684\u8c03\u5ea6\u548c\u8fdb\u7a0b\u542f\u505c\u4e0a\u4e86\uff0c\u56e0\u6b64\u9700\u8981\u589e\u52a0\u6bcf\u4e2atask\u5904\u7406\u7684\u6570\u636e\u5927\u5c0f\u3002\u5efa\u8bae\u4e00\u4e2atask\u5904\u7406\u65f6\u95f4\u4e3a1\u5206\u949f\u3002<\/p>\n<p>\u63a7\u5236\u5355\u4e2atask\u5904\u7406\u65f6\u95f4\u7684\u5927\u5c0f\uff0c\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u914d\u7f6e\u6765\u8c03\u6574\u3002<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; YARN &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<\/p>\n<h2>mapreduce.input.fileinputformat.split.maxsize<br \/>\n\u53ef\u4ee5\u8bbe\u7f6e\u6570\u636e\u5206\u7247\u7684\u6570\u636e\u6700\u5927\u503c\u3002 \u7531\u7528\u6237\u5b9a\u4e49\u7684\u5206\u7247\u5927\u5c0f\u7684\u8bbe\u7f6e\u53ca\u6bcf\u4e2a\u6587\u4ef6block\u5927\u5c0f\u7684\u8bbe\u7f6e\uff0c\u53ef\u4ee5\u8ba1\u7b97\u5206\u7247\u7684\u5927\u5c0f\u3002\u8ba1\u7b97\u516c\u5f0f\u5982\u4e0b\uff1a<br \/>\nsplitSize = Math.max(minSize, Math.min(maxSize, blockSize)) \u5982\u679cmaxSize\u8bbe\u7f6e\u5927\u4e8eblockSize\uff0c\u90a3\u4e48\u6bcf\u4e2ablock\u5c31\u662f\u4e00\u4e2a\u5206\u7247\uff0c\u5426\u5219\u5c31\u4f1a\u5c06\u4e00\u4e2ablock\u6587\u4ef6\u5206\u9694\u4e3a\u591a\u4e2a\u5206\u7247\uff0c\u5982\u679cblock\u4e2d\u5269\u4e0b\u7684\u4e00\u5c0f\u6bb5\u6570\u636e\u91cf\u5c0f\u4e8esplitSize\uff0c\u8fd8\u662f\u8ba4\u4e3a\u5b83\u662f\u72ec\u7acb\u7684\u5206\u7247\u3002<\/h2>\n<p>mapreduce.input.fileinputformat.split.minsize<br \/>\n\u53ef\u4ee5\u8bbe\u7f6e\u6570\u636e\u5206\u7247\u7684\u6570\u636e\u6700\u5c0f\u503c\u3002<br \/>\n0<\/p>\n<h3>12.7.3\u00a0Shuffle\u8c03\u4f18<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>Shuffle\u9636\u6bb5\u662fMapReduce\u6027\u80fd\u7684\u5173\u952e\u90e8\u5206\uff0c\u5305\u62ec\u4e86\u4eceMap task\u5c06\u4e2d\u95f4\u6570\u636e\u5199\u5230\u78c1\u76d8\u4e00\u76f4\u5230Reduce task\u62f7\u8d1d\u6570\u636e\u5e76\u6700\u7ec8\u653e\u5230reduce\u51fd\u6570\u7684\u5168\u90e8\u8fc7\u7a0b\u3002\u8fd9\u4e00\u5757Hadoop\u63d0\u4f9b\u4e86\u5927\u91cf\u7684\u8c03\u4f18\u53c2\u6570\u3002<\/p>\n<p>\u56fe12-1\u00a0Shuffle\u8fc7\u7a0b<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ol>\n<li><strong>Map\u9636\u6bb5\u7684\u8c03\u4f18<\/strong><\/li>\n<\/ol>\n<ul>\n<li>\u5224\u65adMap\u4f7f\u7528\u7684\u5185\u5b58\u5927\u5c0f<\/li>\n<\/ul>\n<p>\u5224\u65adMap\u5206\u914d\u7684\u5185\u5b58\u662f\u5426\u8db3\u591f\uff0c\u4e00\u4e2a\u7b80\u5355\u7684\u529e\u6cd5\u662f\u67e5\u770b\u8fd0\u884c\u5b8c\u6210\u7684job\u7684Counters\u4e2d\uff0c\u5bf9\u5e94\u7684task\u662f\u5426\u53d1\u751f\u8fc7\u591a\u6b21GC\uff0c\u4ee5\u53caGC\u65f6\u95f4\u5360\u603btask\u8fd0\u884c\u65f6\u95f4\u4e4b\u6bd4\u3002\u901a\u5e38\uff0cGC\u65f6\u95f4\u4e0d\u5e94\u8d85\u8fc7task\u8fd0\u884c\u65f6\u95f4\u768410%\uff0c\u5373GC time elapsed (ms)\/CPU time spent (ms)&lt;10%\u3002<\/p>\n<p>\u4e3b\u8981\u901a\u8fc7\u5982\u4e0b\u53c2\u6570\u8fdb\u884c\u8c03\u6574\u3002<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; Yarn &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<br \/>\n\u886812-17\u00a0\u53c2\u6570\u8bf4\u660e<\/p>\n<p>mapreduce.map.memory.mb<br \/>\n\u8bbe\u7f6eMap\u8c03\u5ea6\u5185\u5b58\u3002<br \/>\n4096<br \/>\nmapreduce.map.java.opts<br \/>\n\u8bbe\u7f6eMap\u8fdb\u7a0bJVM\u53c2\u6570\u3002<br \/>\n-Xmx2048M -Djava.net.preferIPv4Stack=true<\/p>\n<p>\u5efa\u8bae\uff1a\u914d\u7f6e\u201cmapreduce.map.java.opts\u201d\u53c2\u6570\u4e2d\u201c-Xmx\u201d\u503c\u4e3a\u201cmapreduce.map.memory.mb\u201d\u53c2\u6570\u503c\u76840.8\u500d\u3002<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/><\/div><\/td><td><div class=\"text codecolorer\">1* \u4f7f\u7528Combiner <br \/>\n2<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u5728Map\u9636\u6bb5\uff0c\u6709\u4e00\u4e2a\u53ef\u9009\u8fc7\u7a0b\uff0c\u5c06\u540c\u4e00\u4e2akey\u503c\u7684\u4e2d\u95f4\u7ed3\u679c\u5408\u5e76\uff0c\u53eb\u505acombiner\u3002\u4e00\u822c\u5c06reduce\u7c7b\u8bbe\u7f6e\u4e3acombiner\u5373\u53ef\u3002\u901a\u8fc7combine\uff0c\u4e00\u822c\u60c5\u51b5\u4e0b\u53ef\u4ee5\u663e\u8457\u51cf\u5c11Map\u8f93\u51fa\u7684\u4e2d\u95f4\u7ed3\u679c\uff0c\u4ece\u800c\u51cf\u5c11shuffle\u8fc7\u7a0b\u7684\u7f51\u7edc\u5e26\u5bbd\u5360\u7528\u3002\u53ef\u901a\u8fc7\u5982\u4e0b\u63a5\u53e3\u4e3a\u4e00\u4e2a\u4efb\u52a1\u8bbe\u7f6eCombiner\u7c7b\u3002<\/p>\n<p>\u886812-18\u00a0Combiner\u8bbe\u7f6e\u63a5\u53e3<\/p>\n<p>org.apache.hadoop.mapreduce.Job<br \/>\npublic void setCombinerClass(Class&lt;? extends Reducer&gt; cls)<br \/>\n\u4e3aJob\u8bbe\u7f6e\u4e00\u4e2acombiner\u7c7b\u3002<\/p>\n<ul>\n<li>\n<p><strong>Copy\u9636\u6bb5\u7684\u8c03\u4f18<\/strong><\/p>\n<\/li>\n<li>\n<p>\u6570\u636e\u662f\u5426\u538b\u7f29<\/p>\n<\/li>\n<\/ul>\n<p>\u5bf9Map\u7684\u4e2d\u95f4\u7ed3\u679c\u8fdb\u884c\u538b\u7f29\uff0c\u5f53\u6570\u636e\u91cf\u5927\u65f6\uff0c\u4f1a\u663e\u8457\u51cf\u5c11\u7f51\u7edc\u4f20\u8f93\u7684\u6570\u636e\u91cf\uff0c\u4f46\u662f\u4e5f\u56e0\u4e3a\u591a\u4e86\u538b\u7f29\u548c\u89e3\u538b\uff0c\u5e26\u6765\u4e86\u66f4\u591a\u7684CPU\u6d88\u8017\u3002\u56e0\u6b64\u9700\u8981\u505a\u597d\u6743\u8861\u3002\u5f53\u4efb\u52a1\u5c5e\u4e8e\u7f51\u7edc\u74f6\u9888\u7c7b\u578b\u65f6\uff0c\u538b\u7f29Map\u4e2d\u95f4\u7ed3\u679c\u6548\u679c\u660e\u663e\u3002\u9488\u5bf9bulkload\u8c03\u4f18\uff0c\u538b\u7f29\u4e2d\u95f4\u7ed3\u679c\u540e\u6027\u80fd\u63d0\u534760%\u5de6\u53f3\u3002<\/p>\n<p>\u914d\u7f6e\u65b9\u6cd5\uff1a\u5c06\u201cmapreduce.map.output.compress\u201d\u53c2\u6570\u503c\u8bbe\u7f6e\u4e3a\u201ctrue\u201d\uff0c\u5c06\u201cmapreduce.map.output.compress.codec\u201d\u53c2\u6570\u503c\u8bbe\u7f6e\u4e3a\u201corg.apache.hadoop.io.compress.SnappyCodec\u201d\u3002<\/p>\n<ul>\n<li><strong>Merge\u9636\u6bb5\u7684\u8c03\u4f18<\/strong><\/li>\n<\/ul>\n<p>\u901a\u8fc7\u8c03\u6574\u5982\u4e0b\u53c2\u6570\u51cf\u5c11reduce\u5199\u78c1\u76d8\u7684\u6b21\u6570\u3002<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; YARN &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<br \/>\n\u886812-19\u00a0\u53c2\u6570\u8bf4\u660e<\/p>\n<p>mapreduce.reduce.merge.inmem.threshold<br \/>\n\u5141\u8bb8\u591a\u5c11\u4e2a\u6587\u4ef6\u540c\u65f6\u5b58\u5728reduce\u5185\u5b58\u91cc\u3002\u5f53\u8fbe\u5230\u8fd9\u4e2a\u9608\u503c\u65f6\uff0creduce\u5c31\u4f1a\u89e6\u53d1mergeAndSpill\uff0c\u5c06\u6570\u636e\u5199\u5230\u786c\u76d8\u4e0a\u3002<br \/>\n1000<br \/>\nmapreduce.reduce.shuffle.merge.percent<br \/>\n\u5f53reduce\u4e2d\u5b58\u653emap\u4e2d\u95f4\u7ed3\u679c\u7684buffer\u4f7f\u7528\u8fbe\u5230\u591a\u5c11\u767e\u5206\u6bd4\u65f6\uff0c\u4f1a\u89e6\u53d1merge\u64cd\u4f5c\u3002<br \/>\n0.66<br \/>\nmapreduce.reduce.shuffle.input.buffer.percent<br \/>\n\u5141\u8bb8map\u4e2d\u95f4\u7ed3\u679c\u5360\u7528reduce\u5806\u5927\u5c0f\u7684\u767e\u5206\u6bd4\u3002<br \/>\n0.70<br \/>\nmapreduce.reduce.input.buffer.percent<br \/>\n\u5f53\u5f00\u59cb\u6267\u884creduce\u51fd\u6570\u65f6\uff0c\u5141\u8bb8map\u6587\u4ef6\u5360reduce\u5806\u5927\u5c0f\u7684\u767e\u5206\u6bd4\u3002 \u5f53map\u6587\u4ef6\u6bd4\u8f83\u5c0f\u65f6\uff0c\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u503c\u8bbe\u7f6e\u62101.0\uff0c\u8fd9\u6837\u53ef\u4ee5\u907f\u514dreduce\u5c06\u62f7\u8d1d\u8fc7\u6765\u7684map\u4e2d\u95f4\u7ed3\u679c\u5199\u78c1\u76d8\u3002<br \/>\n0<\/p>\n<h3>12.7.4\u00a0\u5927\u4efb\u52a1\u7684AM\u8c03\u4f18<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u4efb\u52a1\u573a\u666f\uff1a\u8fd0\u884c\u7684\u4e00\u4e2a\u5927\u4efb\u52a1\uff08map\u603b\u6570\u8fbe\u5230\u4e8610\u4e07\u7684\u89c4\u6a21\uff09\uff0c\u4f46\u662f\u4e00\u76f4\u6ca1\u6709\u8dd1\u6210\u529f\u3002\u7ecf\u8fc7\u67e5\u8be2\uff0c\u53d1\u73b0\u662fApplicationMaster\uff08\u4ee5\u4e0b\u7b80\u79f0AM\uff09\u53cd\u5e94\u7f13\u6162\uff0c\u6700\u7ec8\u8d85\u65f6\u5931\u8d25\u3002<\/p>\n<p>\u6b64\u4efb\u52a1\u7684\u95ee\u9898\u662f\uff0ctask\u6570\u91cf\u53d8\u591a\u65f6\uff0cAM\u7ba1\u7406\u7684\u5bf9\u8c61\u4e5f\u7ebf\u6027\u589e\u957f\uff0c\u56e0\u6b64\u5c31\u9700\u8981\u66f4\u591a\u7684\u5185\u5b58\u6765\u7ba1\u7406\u3002AM\u9ed8\u8ba4\u5206\u914d\u7684\u5185\u5b58\u5806\u5927\u5c0f\u662f1GB\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u901a\u8fc7\u8c03\u5927\u5982\u4e0b\u7684\u53c2\u6570\u6765\u8fdb\u884cAM\u8c03\u4f18\u3002<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; Yarn &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<\/p>\n<p>yarn.app.mapreduce.am.resource.mb<br \/>\n\u8be5\u53c2\u6570\u503c\u5fc5\u987b\u5927\u4e8e\u4e0b\u9762\u53c2\u6570\u7684\u5806\u5927\u5c0f\u3002\u5355\u4f4d\uff1aMB<br \/>\n1536<br \/>\nyarn.app.mapreduce.am.command-opts<br \/>\n\u4f20\u9012\u5230MapReduce ApplicationMaster\u7684JVM\u542f\u52a8\u53c2\u6570\u3002<br \/>\n-Xmx1024m -XX:CMSFullGCsBeforeCompaction=1 -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -verbose:gc<\/p>\n<h3>12.7.5\u00a0\u63a8\u6d4b\u6267\u884c<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5f53\u96c6\u7fa4\u89c4\u6a21\u5f88\u5927\u65f6\uff08\u5982\u51e0\u767e\u4e0a\u5343\u53f0\u8282\u70b9\u7684\u96c6\u7fa4\uff09\uff0c\u4e2a\u522b\u673a\u5668\u51fa\u73b0\u8f6f\u786c\u4ef6\u6545\u969c\u7684\u6982\u7387\u5c31\u53d8\u5927\u4e86\uff0c\u5e76\u4e14\u4f1a\u56e0\u6b64\u5ef6\u957f\u6574\u4e2a\u4efb\u52a1\u7684\u6267\u884c\u65f6\u95f4\uff08\u8dd1\u5b8c\u7684\u4efb\u52a1\u90fd\u5728\u7b49\u51fa\u95ee\u9898\u7684\u673a\u5668\u8dd1\u7ed3\u675f\uff09\u3002\u63a8\u6d4b\u6267\u884c\u901a\u8fc7\u5c06\u4e00\u4e2atask\u5206\u7ed9\u591a\u53f0\u673a\u5668\u8dd1\uff0c\u53d6\u5148\u8fd0\u884c\u5b8c\u7684\u90a3\u4e2a\uff0c\u4f1a\u5f88\u597d\u7684\u89e3\u51b3\u8fd9\u4e2a\u95ee\u9898\u3002\u5bf9\u4e8e\u5c0f\u96c6\u7fa4\uff0c\u53ef\u4ee5\u5c06\u8fd9\u4e2a\u529f\u80fd\u5173\u95ed\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; YARN &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<\/p>\n<p>mapreduce.map.speculative<br \/>\n\u662f\u5426\u5f00\u542fmap\u7684\u63a8\u6d4b\u6267\u884c\u3002true\u8868\u793a\u5f00\u542f\u3002<br \/>\nfalse<br \/>\nmapreduce.reduce.speculative<br \/>\n\u662f\u5426\u5f00\u542freduce\u7684\u63a8\u6d4b\u6267\u884c\u3002true\u8868\u793a\u5f00\u542f\u3002<br \/>\nfalse<\/p>\n<h3>12.7.6\u00a0\u901a\u8fc7\u201cSlow Start\u201d\u8c03\u4f18<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>Slow Start\u7279\u6027\u6307\u5b9aMap\u4efb\u52a1\u5b8c\u6210\u5ea6\u4e3a\u591a\u5c11\u65f6Reduce\u4efb\u52a1\u53ef\u4ee5\u542f\u52a8\uff0c\u8fc7\u65e9\u542f\u52a8Reduce\u4efb\u52a1\u4f1a\u5bfc\u81f4\u8d44\u6e90\u5360\u7528\uff0c\u5f71\u54cd\u4efb\u52a1\u8fd0\u884c\u6548\u7387\uff0c\u4f46\u9002\u5f53\u7684\u63d0\u65e9\u542f\u52a8Reduce\u4efb\u52a1\u4f1a\u63d0\u9ad8Shuffle\u9636\u6bb5\u7684\u8d44\u6e90\u5229\u7528\u7387\uff0c\u63d0\u9ad8\u4efb\u52a1\u8fd0\u884c\u6548\u7387\u3002\u4f8b\u5982\uff1a\u67d0\u96c6\u7fa4\u53ef\u542f\u52a810\u4e2aMap\u4efb\u52a1\uff0cMapReduce\u4f5c\u4e1a\u517115\u4e2aMap\u4efb\u52a1\uff0c\u90a3\u4e48\u5728\u4e00\u8f6eMap\u4efb\u52a1\u6267\u884c\u5b8c\u6210\u540e\u53ea\u52695\u4e2aMap\u4efb\u52a1\uff0c\u96c6\u7fa4\u8fd8\u6709\u5269\u4f59\u8d44\u6e90\uff0c\u5728\u8fd9\u79cd\u573a\u666f\u4e0b\uff0c\u914d\u7f6eSlow Start\u53c2\u6570\u503c\u5c0f\u4e8e1\uff0c\u6bd4\u59820.8\uff0c\u5219Reduce\u5c31\u53ef\u4ee5\u5229\u7528\u96c6\u7fa4\u5269\u4f59\u8d44\u6e90\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; MapReduce &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<\/p>\n<p>mapreduce.job.reduce.slowstart.completedmaps<br \/>\n\u5f53\u591a\u5c11\u5360\u6bd4\u7684Map\u6267\u884c\u5b8c\u540e\u5f00\u59cb\u6267\u884cReduce\u3002\u9ed8\u8ba4100%\u7684Map\u8dd1\u5b8c\u540e\u5f00\u59cb\u8d77Reduce\u3002<br \/>\n1<\/p>\n<h3>12.7.7\u00a0MR job commit\u9636\u6bb5\u4f18\u5316<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u5982\u679c\u4e00\u4e2aMR\u4efb\u52a1\u4f1a\u4ea7\u751f\u5927\u91cf\u7684\u8f93\u51fa\u7ed3\u679c\u6587\u4ef6\uff0c\u90a3\u4e48\u8be5job\u5728\u6700\u540e\u7684commit\u9636\u6bb5\u4f1a\u8017\u8d39\u8f83\u957f\u7684\u65f6\u95f4\u5c06\u6bcf\u4e2atask\u7684\u4e34\u65f6\u8f93\u51fa\u7ed3\u679ccommit\u5230\u6700\u7ec8\u7684\u7ed3\u679c\u8f93\u51fa\u76ee\u5f55\u3002\u7279\u522b\u662f\u5728\u5927\u96c6\u7fa4\u4e2d\uff0c\u5927Job\u7684commit\u8fc7\u7a0b\u4f1a\u4e25\u91cd\u5f71\u54cd\u4efb\u52a1\u7684\u6027\u80fd\u8868\u73b0\u3002<\/p>\n<p>\u9488\u5bf9\u4ee5\u4e0a\u60c5\u51b5\uff0c\u53ef\u4ee5\u901a\u8fc7\u5c06\u4ee5\u4e0b\u53c2\u6570\u201cmapreduce.fileoutputcommitter.algorithm.version\u201d\u914d\u7f6e\u4e3a\u201c2\u201d\uff0c\u6765\u63d0\u5347MR Job commit\u9636\u6bb5\u7684\u6027\u80fd\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; Yarn &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<\/p>\n<p>\u886812-20\u00a0\u53c2\u6570\u8bf4\u660e<\/p>\n<p>mapreduce.fileoutputcommitter.algorithm.version<br \/>\n\u7528\u4e8e\u6307\u5b9aJob\u7684\u6700\u7ec8\u8f93\u51fa\u6587\u4ef6\u63d0\u4ea4\u7684\u7b97\u6cd5\u7248\u672c\uff0c\u53d6\u503c\u4e3a\u201c1\u201d\u6216\u201c2\u201d\u3002 \u8bf4\u660e\uff1a \u7248\u672c2\u4e3a\u5efa\u8bae\u7684\u4f18\u5316\u7b97\u6cd5\u7248\u672c\u3002\u8be5\u7b97\u6cd5\u901a\u8fc7\u8ba9\u4efb\u52a1\u76f4\u63a5\u5c06\u6bcf\u4e2atask\u7684\u8f93\u51fa\u7ed3\u679c\u63d0\u4ea4\u5230\u6700\u7ec8\u7684\u7ed3\u679c\u8f93\u51fa\u76ee\u5f55\uff0c\u4ece\u800c\u51cf\u5c11\u5927\u4f5c\u4e1a\u7684\u8f93\u51fa\u63d0\u4ea4\u65f6\u95f4\u3002<br \/>\n2<\/p>\n<h2>12.8\u00a0Solr<\/h2>\n<h3>12.8.1\u00a0\u7d22\u5f15\u96c6\u5206\u7247\u5212\u5206\u5efa\u8bae<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u8be5\u64cd\u4f5c\u6307\u5bfc\u7cfb\u7edf\u7ba1\u7406\u5458\u901a\u8fc7\u8c03\u6574Solr\u7684\u5b9e\u4f8b\u6570\u548c\u7d22\u5f15\u96c6\uff08Collection\uff09\u7684\u5206\u7247\u6570\uff08Shard\uff09\u6765\u63d0\u9ad8Solr\u7684\u7d22\u5f15\u6027\u80fd\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ul>\n<li>\u8c03\u6574Solr\u7684\u5b9e\u4f8b\u6570<\/li>\n<\/ul>\n<p>\u5982\u67098\u4e2a\u6570\u636e\u8282\u70b9\uff0c\u5efa\u8bae\u6309\u7167\u5982\u4e0b\u65b9\u6848\u8fdb\u884c\u89c4\u5212\uff1a<\/p>\n<p>\u5f53\u7d22\u5f15\u6570\u636e\u5b58\u653e\u5728\u672c\u5730\u78c1\u76d8\u65f6\uff1a<\/p>\n<ul>\n<li>SolrServerAdmin\u89d2\u8272\u5360\u75282\u4e2a\u8282\u70b9\uff0c\u5176\u4e2d\u6bcf\u4e2a\u8282\u70b9\u4e0a\u518d\u90e8\u7f72SolrServerN(N=1\uff5e5)\uff0c\u5171\u8ba112\u4e2a\u5b9e\u4f8b\uff1b<!-- -->\n<ul>\n<li>\u5176\u4f596\u4e2a\u8282\u70b9\uff0c\u6bcf\u4e2a\u8282\u70b9\u5206\u522b\u90e8\u7f72SolrServerN(N=1\uff5e5)\uff0c\u5171\u8ba130\u4e2a\u5b9e\u4f8b\u3002<\/li>\n<li>\u5171\u8ba142\u4e2a\u5b9e\u4f8b<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u5f53\u7d22\u5f15\u6570\u636e\u5b58\u653e\u5728HDFS\u65f6\uff1a<\/p>\n<ul>\n<li>SolrServerAdmin\u89d2\u8272\u5360\u75282\u4e2a\u8282\u70b9\uff0c\u5176\u4e2d\u6bcf\u4e2a\u8282\u70b9\u4e0a\u518d\u90e8\u7f72SolrServerN(N=1\uff5e2)\uff0c\u5171\u8ba13\u4e2a\u5b9e\u4f8b\uff1b<!-- -->\n<ul>\n<li>\u5176\u4f596\u4e2a\u8282\u70b9\uff0c\u6bcf\u4e2a\u8282\u70b9\u5206\u522b\u90e8\u7f72SolrServerN(N=1\uff5e3)\uff0c\u5171\u8ba118\u4e2a\u5b9e\u4f8b\u3002<\/li>\n<li>\u5171\u8ba124\u4e2a\u5b9e\u4f8b<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u8bbe\u7f6e\u6bcf\u4e2a\u5b9e\u4f8b\u5360\u7528\u7cfb\u7edf\u6700\u5927\u5185\u5b58\u4e3a4GB\u3002\u5982\u679c\u5185\u5b58\u8d44\u6e90\u8db3\u591f\uff0c\u6709\u5229\u4e8e\u7d22\u5f15\u6027\u80fd\u63d0\u5347\u3002<\/p>\n<p>\u8c03\u6574\u5185\u5b58\u5360\u7528\u7684\u6b65\u9aa4\u5982\u4e0b\uff1a<\/p>\n<ol>\n<li>\u767b\u5f55FusionInsight Manager\u7cfb\u7edf\u3002<!-- -->\n<ol>\n<li>\u5355\u51fb\u201c\u670d\u52a1\u7ba1\u7406 &gt; Solr &gt; \u670d\u52a1\u914d\u7f6e\u201d\u3002<\/li>\n<li>\u8c03\u8282\u6bcf\u4e2a\u5b9e\u4f8b\u7684\u201cSOLR_GC_OPTS\u201d\u53c2\u6570\uff0c\u201c-Xms2G -Xmx4G\u201d\u90e8\u5206\u8868\u793a\u5360\u7528\u5185\u5b58\u5927\u5c0f\u4e3a2GB\uff5e4GB\u3002\u5c06\u6b64\u53c2\u6570\u503c\u4fee\u6539\u4e3a\u201c-Xms4G -Xmx4G\u201d\u3002<\/li>\n<li>\u5355\u51fb\u201c\u4fdd\u5b58\u914d\u7f6e\u201d\uff0c\u5728\u5f39\u51fa\u7684\u5bf9\u8bdd\u6846\u4e2d\u9009\u62e9\u201c\u91cd\u65b0\u542f\u52a8\u53d7\u5f71\u54cd\u7684\u670d\u52a1\u6216\u5b9e\u4f8b\u3002\u201d\u91cd\u542f\u670d\u52a1\u540e\u751f\u6548\u3002<\/li>\n<\/ol>\n<\/li>\n<li>\u8c03\u6574Solr\u7d22\u5f15\u96c6\u7684\u5206\u7247\u6570<\/li>\n<\/ol>\n<p>\u5728\u521b\u5efaHDFS\u6570\u636e\u548cHBase\u6570\u636e\u7684\u5168\u6587\u68c0\u7d22\u7d22\u5f15\u96c6\u65f6\uff08\u5177\u4f53\u64cd\u4f5c\u53ef\u53c2\u89c1\u201c\u4e1a\u52a1\u64cd\u4f5c\u6307\u5357 &gt; Solr &gt; \u4e1a\u52a1\u5e38\u89c1\u64cd\u4f5c\u201d\uff09\uff0c\u5efa\u8bae\u6309\u7167\u4ee5\u4e0b\u65b9\u6848\u8fdb\u884c\u89c4\u5212\uff1a<\/p>\n<p>\u8bbe\u7f6e\u6bcf\u4e2a\u5b9e\u4f8b\u5bf9\u5e941\u4e2aShard\uff0c\u6bcf\u4e2ashard\u5305\u542b\u4e24\u4e2areplica\uff08\u5373\u526f\u672c\u56e0\u5b50\u4e3a2\uff09\uff0c\u5f53\u8981\u6c42\u526f\u672c\u6570\u4e3a2\u65f6\uff0c\u521b\u5efa\u5305\u542b21\u4e2ashard\u7684collection\uff0c\u6bcf\u4e2ashard\u4e24\u4e2a\u526f\u672c\u5206\u5e03\u4e8e\u4e24\u4e2a\u4e0d\u540c\u8282\u70b9\u5b9e\u4f8b\uff0c\u8fd9\u683742\u4e2a\u5b9e\u4f8b\u5206\u522b\u5305\u542b\u4e00\u4e2areplica\u3002<\/p>\n<p>\u6839\u636e\u4ee5\u4e0a\u65b9\u6848\uff0c\u6bcf\u4e2a\u7d22\u5f15\u96c6\u5305\u542b42\u4e2aShard\uff08\u4ee5\u90e8\u7f7242\u4e2a\u5b9e\u4f8b\u4e3a\u4f8b\uff09\u3002Shard\u6570\u91cf\u8d8a\u5927\uff0c\u6709\u52a9\u4e8e\u67e5\u8be2\u6216\u7d22\u5f15\u7684\u6027\u80fd\u63d0\u5347\uff0c\u4f46\u540c\u65f6\u4e5f\u4f1a\u589e\u52a0\u670d\u52a1\u7684\u4ea4\u4e92\u64cd\u4f5c\uff0c\u6d88\u8017\u66f4\u591a\u7cfb\u7edf\u8d44\u6e90\u3002<\/p>\n<p>\u5f53\u7d22\u5f15\u6570\u636e\u5b58\u653e\u5728\u672c\u5730\u78c1\u76d8\u65f6\uff0c\u540c\u6837\u53c2\u8003\u4ee5\u4e0a\u89c4\u5219\u8fdb\u884c\u89c4\u5212\uff0c\u4e5f\u5c31\u662f\u6bcf\u4e2a\u7d22\u5f15\u96c6\u5305\u542b24\u4e2ashard(\u4ee5\u90e8\u7f7224\u4e2a\u5b9e\u4f8b\u4e3a\u4f8b)\u3002<\/p>\n<h3>12.8.2\u00a0Solr\u516c\u5171\u8bfb\u5199\u8c03\u4f18\u5efa\u8bae<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u8be5\u64cd\u4f5c\u6307\u5bfc\u7cfb\u7edf\u7ba1\u7406\u5458\u5bf9Solr\u53ef\u8fdb\u884c\u7684\u516c\u5171\u8bfb\u5199\u6027\u80fd\u8c03\u4f18\u3002<\/p>\n<p>\u524d\u63d0\u6b65\u9aa4<\/p>\n<p>\u5df2\u5b89\u88c5Solr\u670d\u52a1\u7684\u5ba2\u6237\u7aef\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u5f53Solr\u7d22\u5f15\u6570\u636e\u5b58\u653e\u5728\u672c\u5730\u78c1\u76d8\u6216\u8005HDFS\u65f6\uff0c\u53ef\u4ee5\u4ece\u4ee5\u4e0b\u51e0\u4e2a\u65b9\u9762\u8fdb\u884c\u8c03\u4f18\u914d\u7f6e\uff1a<\/p>\n<p>Solr\u4f5c\u4e3a\u5168\u6587\u68c0\u7d22\u670d\u52a1\u5668\u65f6\uff0c\u4eceSolr\u81ea\u8eab\u76f8\u5173\u914d\u7f6e\uff08JVM\u3001Schema\u3001Solrconfig\uff09\u65b9\u9762\u8fdb\u884c\u8c03\u4f18\u3002<\/p>\n<ol>\n<li>Schema\u7684\u914d\u7f6e\u4f18\u5316<\/li>\n<\/ol>\n<ul>\n<li>\n<p>uniqueKey\u5b9a\u4e49\u4e3along\u7c7b\u578b\u00a0\u8bf4\u660e\uff1a<\/p>\n<\/li>\n<li>\n<p>long\u7c7b\u578b\u7684\u67e5\u8be2\u6027\u80fd\u4f18\u4e8estring\u7c7b\u578b\uff0c\u5982\u679c\u9700\u8981\u5b9a\u4e49\u4e3astring\u7c7b\u578b\uff0c\u53ef\u4ee5\u5728\u4e1a\u52a1\u5c42\u5efa\u7acblong\u5230string\u7684\u6620\u5c04\u3002<\/p>\n<\/li>\n<li>\n<p>\u5efa\u8baeunqiueKey\u5b57\u6bb5\u914d\u7f6erequired=&quot;true&quot;\u3002<br \/>\n* \u5efa\u8baeuniqueKey\u5b57\u6bb5\u914d\u7f6edocValues=&quot;true&quot;\u3002<br \/>\n* \u4e3a\u4e86\u83b7\u5f97\u66f4\u597d\u7684\u67e5\u8be2\u6027\u80fd\uff0c\u5efa\u8bae\u67e5\u8be2\u65f6\u663e\u5f0f\u5730\u6307\u5b9a\u8fd4\u56de\u5b57\u6bb5\u4e3auniqueKey\u5b57\u6bb5\u3002<\/p>\n<ul>\n<li>\u9700\u8981\u6392\u5e8f\u3001\u7edf\u8ba1\u7684\u5b57\u6bb5\u914d\u7f6e\u4e3adocValues=&quot;true&quot;\uff0c\u53ef\u4ee5\u6709\u6548\u8282\u7701\u5185\u5b58\u4f7f\u7528\u00a0\u8bf4\u660e\uff1a<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u914d\u7f6edocValues=&quot;true&quot;\u540e\uff0c\u4e0d\u9700\u8981\u518d\u914d\u7f6estored=&quot;true&quot;<\/p>\n<ul>\n<li>\u4f18\u5316\u7d22\u5f15\u65b9\u6848\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539\u201csolrconfig.xml\u201d\u6587\u4ef6\u5185\u5bb9\u6765\u5b9e\u73b0\u4ee5\u4e0b\u8c03\u4f18\u6548\u679c\u3002<\/li>\n<\/ul>\n<p>\u63d0\u9ad8\u7d22\u5f15\u901f\u5ea6\uff0c\u52a0\u5927\u7d22\u5f15\u7ebf\u7a0b\u6570<br \/>\n<!-- -->&lt;maxIndexingThreads&gt;<!-- -->${solr.maxIndexingThreads:16}<!-- -->&lt;\/maxIndexingThreads&gt;<!-- --><br \/>\n\u589e\u5927\u6587\u6863\u7d22\u5f15\u7f13\u5b58<br \/>\n<!-- -->&lt;ramBufferSizeMB&gt;<!-- -->1024<!-- -->&lt;\/ramBufferSizeMB&gt;<!-- --><br \/>\n\u589e\u5927\u7d22\u5f15\u6bb5\u5408\u5e76\u56e0\u5b50<br \/>\n<!-- -->&lt;mergeFactor&gt;<!-- -->20<!-- -->&lt;\/mergeFactor&gt;<!-- --><br \/>\n\u52a0\u5927\u7d22\u5f15\u81ea\u52a8\u786c\u63d0\u4ea4\u65f6\u95f4<br \/>\n<!-- -->&lt;maxTime&gt;<!-- -->${solr.autoCommit.maxTime:30000}<!-- -->&lt;\/maxTime&gt;<!-- --><br \/>\n\u589e\u5927\u7d22\u5f15\u7684\u81ea\u52a8\u8f6f\u63d0\u4ea4\u65f6\u95f4<br \/>\n<!-- -->&lt;maxTime&gt;<!-- -->${solr.autoSoftCommit.maxTime:60000}<!-- -->&lt;\/maxTime&gt;<!-- --><br \/>\n\u57fa\u4e8edocValues\u83b7\u53d6uniqueKey\u7684\u503c<br \/>\n<!-- -->&lt;useDocValueGetField&gt;<!-- -->true<!-- -->&lt;\/useDocValueGetField&gt;<!-- --><br \/>\n\u5bf9docId\u8fdb\u884c\u6392\u5e8f\uff0c\u4f18\u5148\u987a\u5e8f\u8bfb\u53d6\u78c1\u76d8<br \/>\n<!-- -->&lt;sortDocIdBeforeGetDoc&gt;<!-- -->true<!-- -->&lt;\/sortDocIdBeforeGetDoc&gt;<!-- --><br \/>\n\u7f13\u5b58docId\uff0c\u907f\u514d\u4e8c\u6b21\u8bfb\u53d6\u78c1\u76d8<br \/>\n<!-- -->&lt;useQuickFirstMatch&gt;<!-- -->true<!-- -->&lt;\/useQuickFirstMatch&gt;<!-- -->\u00a0\u8bf4\u660e\uff1auseDocValueGetField\u7684\u4f7f\u7528\u573a\u666f\uff1a<\/p>\n<ul>\n<li>\u8fd4\u56de\u5b57\u6bb5\uff08fl\uff09\u4e3auniqueKey<!-- -->\n<ul>\n<li>uniqueKey\u4e3aNumberic\u7c7b\u578b\uff08long\/int\/float\/double\uff09<\/li>\n<li>uniqueKey\u914d\u7f6edocValues=true<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>useQuickFirstMatch\u7684\u4f7f\u7528\u573a\u666f\uff1a<\/p>\n<ul>\n<li>\n<p>\u7d22\u5f15\u5165\u5e93\u540e\u4e0d\u518d\u66f4\u6539\uff08\u5220\u9664\/\u5408\u5e76\u64cd\u4f5c\u7b49\uff09<\/p>\n<\/li>\n<li>\n<p>\u4f18\u5316\u67e5\u8be2\u65b9\u6848\u7f13\u5b58\u5728Solr\u4e2d\u5145\u5f53\u4e86\u4e00\u4e2a\u975e\u5e38\u91cd\u8981\u7684\u89d2\u8272\uff0cSolr\u4e2d\u4e3b\u8981\u5305\u62ec\u4ee5\u4e0b3\u79cd\u7f13\u5b58\uff1a<\/p>\n<\/li>\n<li>\n<p>Filter cache\uff08\u8fc7\u6ee4\u5668\u7f13\u5b58\uff09\uff0c\u7528\u4e8e\u4fdd\u5b58\u8fc7\u6ee4\u5668\uff08fq\u53c2\u6570\uff09\u548c\u5c42\u9762\u641c\u7d22\u7684\u7ed3\u679c\uff1b<\/p>\n<ul>\n<li>Document cache\uff08\u6587\u6863\u7f13\u5b58\uff09\uff0c\u7528\u4e8e\u4fdd\u5b58lucene\u6587\u6863\u5b58\u50a8\u7684\u5b57\u6bb5\uff1b<\/li>\n<li>Query result\uff08\u67e5\u8be2\u7f13\u5b58\uff09\uff0c\u7528\u4e8e\u4fdd\u5b58\u67e5\u8be2\u7684\u7ed3\u679c\u3002\u00a0\u8bf4\u660e\uff1a<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>Solr\u4e2d\u8fd8\u5305\u542blucene\u5185\u90e8\u7f13\u5b58\uff0c\u8be5\u7f13\u5b58\u7528\u6237\u65e0\u6cd5\u8c03\u63a7\u3002<\/p>\n<p>\u901a\u8fc7\u8c03\u6574\u8fd93\u79cd\u7f13\u5b58\uff0c\u53ef\u4ee5\u5bf9Solr\u7684\u641c\u7d22\u5b9e\u4f8b\u8fdb\u884c\u8c03\u4f18\u3002\u5728\u8c03\u6574\u53c2\u6570\u524d\uff0c\u9700\u8981\u4e8b\u5148\u5f97\u5230Solr\u5b9e\u4f8b\u4e2d\u7684\u4ee5\u4e0b\u4fe1\u606f\uff1a<\/p>\n<ul>\n<li>\u7d22\u5f15\u4e2d\u6587\u6863\u7684\u6570\u91cf\uff1a\u5355\u51fb\u201c\u670d\u52a1\u7ba1\u7406 &gt; Solr\u201d\uff0c\u5728\u201cSolr WebUI\u201d\u4e0b\u5355\u51fb\u4efb\u610f\u201cSolrServerAdmin(XX)\u201d\u8fdb\u5165Solr Admin\u9875\u9762\u3002\u5728\u201cCore Selector\u201d\u4e2d\u9009\u62e9\u7d22\u5f15\u6570\u636e\u96c6Collection\u4e0b\u7684\u4efb\u610f\u4e00\u4e2acore\uff0c\u5355\u51fb\u201cQuery &gt; Execute Query\u201d\uff0c\u201cnumFound\u201d\u9879\u4e3a\u8be5\u53c2\u6570\u503c\uff1b<!-- -->\n<ul>\n<li>\u8fc7\u6ee4\u5668\u7684\u6570\u91cf\uff1a\u7528\u6237\u671f\u671b\u503c\uff08\u5982\uff1a200\uff09\uff1b<\/li>\n<li>\u4e00\u6b21\u67e5\u8be2\u8fd4\u56de\u6700\u5927\u7684\u6587\u6863\u6570\u91cf\uff1a\u7528\u6237\u671f\u671b\u503c\uff08\u5982\uff1a100\uff09\uff1b<\/li>\n<li>\u4e0d\u540c\u67e5\u8be2\u548c\u4e0d\u540c\u6392\u5e8f\u7684\u4e2a\u6570\uff1a\u7528\u6237\u671f\u671b\u503c\uff08\u5982\uff1a500\uff09\uff1b<\/li>\n<li>\u6bcf\u6b21\u67e5\u8be2\u5b57\u6bb5\u6570\uff1a\u7528\u6237\u671f\u671b\u503c\uff08\u5982\uff1a3\uff09\uff1b<\/li>\n<li>\u5b9e\u4f8b\u67e5\u8be2\u7684\u5e76\u53d1\u6570\uff1a\u7528\u6237\u671f\u671b\u503c\uff08\u5982\uff1a10\uff09\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539\u201csolrconfig.xml\u201d\u6587\u4ef6\u5185\u5bb9\uff08\u4fee\u6539\u65b9\u6cd5\u53c2\u80033\uff09\u914d\u7f6e\u7f13\u5b58\uff1a<\/p>\n<p>\u8fc7\u6ee4\u5668\u7f13\u5b58<br \/>\n<!-- -->&lt;filterCache class=&quot;solr.FastLRUCache&quot; size=&quot;200&quot; initialSize=&quot;200&quot; autowarmCount=&quot;50&quot;\/&gt;<!-- --> \u201csize\u201d\u548c\u201cinitialSize\u201d\u503c\u4e3a\u7f13\u5b58document id\u7684\u6570\u91cf\u3002 \u201cautowarmCount\u201d\u4e3a\u201cinitialSiz\u201d\u503c\u76841\/4\u3002 \u6839\u636e\u5b9e\u9645\u573a\u666f\u5408\u7406\u8bbe\u7f6e\uff0c\u8fc7\u5927\u4f1a\u5360\u7528\u5927\u91cf\u5185\u5b58\u3002<br \/>\n\u67e5\u8be2\u7ed3\u679c\u7f13\u5b58<br \/>\n<!-- -->&lt;queryResultCache class=&quot;solr.FastLRUCache&quot; size=&quot;3000&quot; initialSize=&quot;3000&quot; autowarmCount=&quot;750&quot;\/&gt;<!-- --> \u201csize\u201d\u548c\u201cinitialSize\u201d\u7684\u503c =\u00a0\u4e0d\u540c\u67e5\u8be2\u548c\u4e0d\u540c\u6392\u5e8f\u7684\u4e2a\u6570\u00d7\u6bcf\u6b21\u67e5\u8be2\u5b57\u6bb5\u6570\u00d72\u3002 \u201cautowarmCount\u201d\u503c\u4e3a\u201cinitialSize\u201d\u503c\u76841\/4\u3002<br \/>\n\u6587\u6863\u7f13\u5b58<br \/>\n<!-- -->&lt;documentCache class=&quot;solr.FastLRUCache&quot; size=&quot;1000&quot; initialSize=&quot;1000&quot;\/&gt;<!-- --> \u201csize\u201d\u548c\u201cinitialSize\u201d\u7684\u503c =\u00a0\u4e00\u6b21\u67e5\u8be2\u8fd4\u56de\u6700\u5927\u7684\u6587\u6863\u6570\u91cf\u00d7\u5b9e\u4f8b\u67e5\u8be2\u7684\u5e76\u53d1\u6570\u3002<\/p>\n<h3>12.8.3\u00a0Solr over HBase\u8c03\u4f18\u5efa\u8bae<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u8be5\u64cd\u4f5c\u6307\u5bfc\u7cfb\u7edf\u7ba1\u7406\u5458\u5728\u4f7f\u7528Solr over HBase\u76f8\u5173\u529f\u80fd\u65f6\uff0c\u5bf9\u73af\u5883\u914d\u7f6e\u8fdb\u884c\u8c03\u4f18\u3002<\/p>\n<p>\u524d\u63d0\u6761\u4ef6<\/p>\n<p>\u5df2\u6210\u529f\u5b89\u88c5HDFS\u3001Solr\u3001Yarn\u3001HBase\u670d\u52a1\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u4f7f\u7528Solr over HBase\u76f8\u5173\u529f\u80fd\u65f6\uff0c\u53ef\u4ee5\u4ece\u4ee5\u4e0b\u51e0\u4e2a\u65b9\u9762\u8fdb\u884c\u8c03\u4f18\u914d\u7f6e\uff1a<\/p>\n<ul>\n<li>\u64cd\u4f5c\u7cfb\u7edf\u4f18\u5316<\/li>\n<\/ul>\n<p>\u5982\u679cSolr\u7d22\u5f15\u5728HDFS\u4e0a\uff0c\u53c2\u8003\u300aSolr over HDFS\u8c03\u4f18\u5efa\u8bae\u300b\u7ae0\u8282\u64cd\u4f5c\u6b65\u9aa4\u4e2d\u64cd\u4f5c\u7cfb\u7edf\u4f18\u5316\u5c0f\u8282\u8fdb\u884c\u914d\u7f6e\u3002<\/p>\n<ul>\n<li>\u5b9e\u65f6\u7d22\u5f15\u76f8\u5173\u8c03\u4f18\u5efa\u8bae<\/li>\n<\/ul>\n<ol>\n<li>\u4fee\u6539collection\u914d\u7f6e\u96c6\u7684\u201csolrconfig.xml\u201d\u914d\u7f6e\u6587\u4ef6\u4e2d\uff0c**<!-- -->&lt;autoSoftCommit&gt;<!-- -->**\u914d\u7f6e\u9879\uff0c\u6839\u636e\u4f7f\u7528\u573a\u666f\u5c3d\u91cf\u8bbe\u7f6e\u5927\u4e00\u4e9b\uff0c\u8bbe\u7f6e\u8d8a\u5927\u7d22\u5f15\u6548\u7387\u8d8a\u9ad8\u3002<\/li>\n<li>\u4fee\u6539HBase\u670d\u52a1\u914d\u7f6e\uff0c\u7136\u540e\u91cd\u542fHBase\u670d\u52a1\uff1a<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p>\u201creplication.source.nb.capacity\u201d\uff1a5000\uff08HBase\u96c6\u7fa4\u6bcf\u6b21\u5411HBaseIndexer\u53d1\u9001\u7684entry\u6700\u5927\u7684\u4e2a\u6570\uff0c\u63a8\u83505000\u3002\u53ef\u6839\u636e\u96c6\u7fa4\u89c4\u6a21\u505a\u51fa\u9002\u5f53\u8c03\u6574\uff0c\u6839\u636eHBaseIndexer\u90e8\u7f72\u60c5\u51b5\u9002\u5f53\u589e\u5927\uff09\u3002<\/p>\n<p>\u201creplication.source.size.capacity\u201d\uff1a4194304\uff08HBase\u6bcf\u6b21\u5411HBaseIndexer\u53d1\u9001\u7684entry\u5305\u7684\u6700\u5927\u503c\u5927\u5c0f\uff0c\u4e0d\u63a8\u8350\u8fc7\u5927\uff09\u3002<\/p>\n<p>\u00a0<\/p>\n<ol>\n<li>\u4fee\u6539HDFS\u670d\u52a1\u914d\u7f6e\uff0c\u7136\u540e\u91cd\u542fHDFS\u670d\u52a1\uff1a<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p>\u201chadoop.rpc.protection\u201d\uff1aauthentication\uff08\u5173\u95ed\u6570\u636e\u4f20\u8f93\u52a0\u5bc6\uff0c\u9ed8\u8ba4\u4e3aprivacy\uff09\u3002<\/p>\n<p>\u201cipc.server.handler.queue.size\u201d\uff1a\u961f\u5217\u4e2d\u5141\u8bb8\u7684\u6bcf\u4e2a\u5904\u7406\u7a0b\u5e8f\u53ef\u5904\u7406\u7684\u8c03\u7528\u6570\uff0c\u6839\u636e\u96c6\u7fa4\u73af\u5883\u9002\u5f53\u8c03\u6574\u3002<\/p>\n<p>\u201cdfs.namenode.handler.count\u201d\uff1aNameNode\u7684\u670d\u52a1\u5668\u7ebf\u7a0b\u6570\uff0c\u6839\u636e\u96c6\u7fa4\u73af\u5883\u9002\u5f53\u8c03\u6574\u3002<\/p>\n<p>\u201cdfs.namenode.service.handler.count\u201d\uff1aNameNode\u7684\u670d\u52a1\u5668\u7ebf\u7a0b\u6570\uff0c\u6839\u636e\u96c6\u7fa4\u73af\u5883\u9002\u5f53\u8c03\u6574\u3002<\/p>\n<p>\u201cdfs.datanode.handler.count\u201d\uff1aDataNode\u7684\u670d\u52a1\u7ebf\u7a0b\u6570\uff0c\u6839\u636e\u96c6\u7fa4\u73af\u5883\u9002\u5f53\u8c03\u6574\u3002<\/p>\n<p>\u00a0<\/p>\n<ol>\n<li>\u4fee\u6539HBaseIndexer\u670d\u52a1\u914d\u7f6e\uff0c\u7136\u540e\u91cd\u542fHBaseIndexer\u5b9e\u4f8b\uff1a<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p>\u201chbaseindexer.indexer.threads\u201d\uff1a50\uff08\u9ed8\u8ba4\u503c\u4e3a20\uff0cHBaseIndexer\u5b9e\u4f8b\u8fdb\u884c\u7d22\u5f15\u64cd\u4f5c\u65f6\u542f\u52a8\u7684\u5e76\u53d1\u7ebf\u7a0b\u6570\u91cf\uff09\u3002<\/p>\n<p>\u8c03\u6574\u00a0\u201cGC_OPTS\u201d\u81f34G\uff0c\u5185\u5b58\u7a7a\u95f4\u5145\u8db3\uff0c\u53ef\u4ee5\u8003\u8651\u9002\u5f53\u589e\u52a0\u3002<\/p>\n<p>\u00a0<\/p>\n<ul>\n<li>\u6279\u91cf\u7d22\u5f15\u3001\u589e\u91cf\u7d22\u5f15\u76f8\u5173\u8c03\u4f18\u5efa\u8bae<\/li>\n<\/ul>\n<ol>\n<li>SolrServer\u7684GC\u53c2\u6570\u914d\u7f6e\uff08\u5982\u679c\u5185\u5b58\u5145\u8db3\u53ef\u4ee5\u8003\u8651\u589e\u5927\uff09\uff1a**-Xmx8G\u00a0**<strong>-Xms8G<\/strong><\/li>\n<li>Yarn\u7684\u914d\u7f6e\uff0c\u4fee\u6539\u540e\u91cd\u542fYarn\u670d\u52a1\uff1a<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p>\u201cmapreduce.reduce.memory.mb\u201d\uff1a8192\uff08\u6839\u636e\u8282\u70b9\u914d\u7f6e\u8fdb\u884c\u4fee\u6539\uff09<\/p>\n<p>\u201cyarn.resourcemanager.scheduler.class\u201d\uff1aorg.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler<\/p>\n<p>\u00a0<\/p>\n<ol>\n<li>\u4fee\u6539HBaseIndexer\u7684\u914d\u7f6e\u6587\u4ef6\uff0c\u8fdb\u5165\u201cSolr\u5ba2\u6237\u7aef\u7684\u5b89\u88c5\u76ee\u5f55\/hbase-indexer\/\u201d\uff0c\u6267\u884c\u4e00\u4e0b\u547d\u4ee4\u4fee\u6539\u914d\u7f6e\u6587\u4ef6\uff1a<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p><strong>vi \/opt\/client\/Solr\/hbase-indexer\/conf\/hbase-indexer-site.xml<\/strong><\/p>\n<p>\u8bbe\u7f6e\u201chbase-indexer-site.xml\u201d\u90e8\u5206\u4ee5\u4e0b\u51e0\u4e2a\u53c2\u6570\u7684\u53c2\u6570\u503c\uff08\u5982\u67d0\u4e9b\u53c2\u6570\u9879\u4e0d\u5b58\u5728\uff0c\u53ef\u624b\u52a8\u6dfb\u52a0\uff09\uff1a<br \/>\n\u886812-21\u00a0\u201chbase-indexer-site.xml\u201d\u914d\u7f6e\u6587\u4ef6\u4fee\u6539<\/p>\n<p>solr.record.writer.batch.size<br \/>\n500<br \/>\nsolr.record.writer.max.queues.size<br \/>\n300<br \/>\nsolr.record.writer.num.threads<br \/>\n5<br \/>\nsolr.record.writer.maxSegments<br \/>\n5<\/p>\n<p><strong>vi \/opt\/client\/Solr\/hbase-indexer\/conf\/yarn-site.xml<\/strong><\/p>\n<p>\u8bbe\u7f6e\u201cyarn-site.xml\u201d\u90e8\u5206\u4ee5\u4e0b\u51e0\u4e2a\u53c2\u6570\u7684\u53c2\u6570\u503c\uff08\u5982\u67d0\u4e9b\u53c2\u6570\u9879\u4e0d\u5b58\u5728\uff0c \u53ef\u624b\u52a8\u6dfb\u52a0\uff09\uff1a<br \/>\n\u886812-22\u00a0\u201cyarn-site.xml\u201d\u914d\u7f6e\u6587\u4ef6\u4fee\u6539<\/p>\n<p>mapreduce.map.speculative<br \/>\nfalse<br \/>\nmapreduce.reduce.speculative<br \/>\nfalse<\/p>\n<p>\u00a0<\/p>\n<h3>12.8.4\u00a0Solr over HDFS\u8c03\u4f18\u5efa\u8bae<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u8be5\u64cd\u4f5c\u6307\u5bfc\u7cfb\u7edf\u7ba1\u7406\u5458\u5728\u4f7f\u7528Solr over HDFS\u76f8\u5173\u529f\u80fd\u65f6\uff0c\u5bf9\u73af\u5883\u914d\u7f6e\u8fdb\u884c\u8c03\u4f18\u3002<\/p>\n<p>\u524d\u63d0\u6761\u4ef6<\/p>\n<p>\u5df2\u6210\u529f\u5b89\u88c5HDFS\u3001Solr\u3001Yarn\u670d\u52a1\uff0c\u4e14FusionInsight Manager\u9875\u9762\u5185Solr\u670d\u52a1\u7684\u670d\u52a1\u914d\u7f6e\u53c2\u6570\u201cINDEX_STORED_ON_HDFS\u201d\u4e3a\u201cTRUE\u201d\u3002<\/p>\n<p>\u5b8c\u6210\u4f7f\u7528Solr over HDFS\u7684\u51c6\u5907\u5de5\u4f5c\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u4f7f\u7528Solr over HDFS\u65f6\uff0c\u53ef\u4ee5\u9009\u62e9\u4ece\u4ee5\u4e0b\u51e0\u4e2a\u65b9\u9762\u8fdb\u884c\u8c03\u4f18\u914d\u7f6e\uff1a<\/p>\n<ul>\n<li>\u78c1\u76d8\u548c\u7f51\u7edc\u89c4\u5212<\/li>\n<\/ul>\n<ol>\n<li>\u78c1\u76d8\u5212\u5206\u7684\u65f6\u5019\u8981\u6ce8\u610f\uff0cZooKeeper\u5355\u72ec\u5360\u7528\u4e00\u4e2a\u78c1\u76d8\u6216\u4e00\u4e2a\u78c1\u76d8\u5206\u533a\uff0c\u5426\u5219\u5f53\u6570\u636e\u91cf\u8fc7\u5927\uff0c\u5982\u679c\u548cHDFS\u5171\u78c1\u76d8\uff0c\u9891\u7e41\u7684\u6570\u636e\u8bbf\u95ee\u548c\u914d\u7f6e\u96c6\u8bbf\u95ee\uff0c\u4f1a\u5bfc\u81f4ZooKeeper\u505c\u6b62\u54cd\u5e94\u3002<\/li>\n<li>HDFS\u91c7\u7528\u591a\u5757\u5355\u76d8\u6216\u8005\u591a\u4e2aRAID0\u5206\u522b\u6302\u8f7d\u591a\u4e2a\u6570\u636e\u76ee\u5f55\uff0cMR\u4efb\u52a1\u5904\u7406\u6570\u636e\u91cf\u5f88\u5927\u65f6\u78c1\u76d8IO\u4f1a\u6210\u4e3a\u74f6\u9888\u3002<\/li>\n<li>\u7f51\u7edc\u7ec4\u7f51\u6a21\u5f0f\uff0c\u5f53\u524d\u73af\u5883\u4e3a\u591a\u7f51\u5361\u7ed1\u5b9a\u6a21\u5f0fbond0\uff0c\u7f51\u7edc\u7ec4\u7f51\u6a21\u5f0f\u53ef\u6839\u636e\u5b9e\u9645\u60c5\u51b5\u800c\u5b9a\uff0c\u4e0d\u4e00\u5b9a\u8981\u6c42\u4e3a\u591a\u7f51\u5361\u7ed1\u5b9a\u6a21\u5f0f\u3002<\/li>\n<\/ol>\n<ul>\n<li>\u64cd\u4f5c\u7cfb\u7edf\u4f18\u5316<\/li>\n<\/ul>\n<p>\u6bcf\u53f0\u4e3b\u673a\u4e0a\u90fd\u8981\u6267\u884c\uff0c\u7531\u4e8eSolr\u5728\u6267\u884c\u8bfb\u53d6HDFS\u65f6\uff0c\u4f1a\u751f\u6210\u5f88\u591a\u4e2a\u4e34\u65f6\u7aef\u53e3\uff08\u8fde\u63a5\u672c\u5730DataNode\u3002\u767b\u5f55FusionInsight Manager\uff0c\u5355\u51fb\u201c\u670d\u52a1\u7ba1\u7406 &gt; HDFS &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u67e5\u770b\u201cdfs.datanode.port\u201d\uff0c\u5176\u503c\u4e3a\u201cDataNode Port\u201d\u3002\u6267\u884c<strong>netstat -anp | grep\u00a0<em><strong>DataNode Port<\/strong><\/em>\u00a0| wc -l<\/strong>\u547d\u4ee4\uff0c\u7ed3\u679c\u5927\u4e8e4096\u65f6\u3002\uff09\uff0c\u51fa\u73b0TIME_WAIT\uff0c\u6700\u7ec8\u5bfc\u81f4\u4efb\u52a1\u5931\u8d25\uff0c\u4e3a\u907f\u514d\u6b64\u79cd\u60c5\u51b5\uff0c\u9700\u8981\u5982\u4e0b\u8bbe\u7f6e<\/p>\n<p>\u4f7f\u7528PuTTY\u5ba2\u6237\u7aef\uff0c\u767b\u5f55\u6bcf\u4e2a\u8282\u70b9\uff0c\u8fdb\u5165Solr\u5ba2\u6237\u7aef\u6240\u5728\u76ee\u5f55\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff1a<\/p>\n<p><strong>vi \/etc\/sysctl.conf<\/strong><\/p>\n<p>\u6dfb\u52a0\u4ee5\u4e0b\u5185\u5bb9\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/>6<br \/>7<br \/>8<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2net.ipv4.tcp_syncookies = 1<br \/>\n3net.ipv4.tcp_tw_reuse = 1<br \/>\n4net.ipv4.tcp_tw_recycle = 1<br \/>\n5net.ipv4.tcp_fin_timeout = 30<br \/>\n6net.ipv4.tcp_timestamps = 1<br \/>\n7net.ipv4.tcp_tw_recycle = 1<br \/>\n8<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u4fdd\u5b58\u9000\u51fa\u540e\uff0c\u6267\u884c<\/p>\n<p><strong>sysctl -p<\/strong><\/p>\n<ul>\n<li>Solr\u5b9e\u4f8b\u90e8\u7f72\uff0c\u7d22\u5f15\u5b58\u653e\u5230HDFS\u4e0a\u65f6\uff0c\u786e\u4fdd\u6240\u6709Solr\u5b9e\u4f8b\u540cDataNode\u90e8\u7f72\u5728\u76f8\u540c\u7684\u8282\u70b9\u4e0a\u3002<\/li>\n<li>\u5bf9Solr\u63d0\u4ea4\u7684MapReduce\u4efb\u52a1\u7684\u6027\u80fd\u4f18\u5316\uff0c\u8bf7\u53c2\u8003\u8282\u70b9\u914d\u7f6e\u8c03\u4f18\u548cJVM\u53c2\u6570\u4f18\u5316\u3002<\/li>\n<\/ul>\n<p>\u53ef\u53c2\u8003\u5982\u4e0b\u5efa\u8bae\uff0c\u8bbe\u7f6eYarn\u670d\u52a1\u7684\u914d\u7f6e\u53c2\u6570\uff0c\u7136\u540e\u91cd\u542fYarn\u670d\u52a1\uff1a<\/p>\n<p>\u201cmapreduce.task.timeout\u201d\uff1a1800000\uff08\u539f\u59cb\u503c\u4e3a60000\uff0c\u5904\u7406\u5927\u6570\u636e\u91cf\u65f6\u53ef\u4ee5\u9002\u5f53\u8c03\u5927\uff09\u3002<\/p>\n<p>\u201cyarn.nodemanager.resource.cpu-vcores\u201d\uff1a24\uff08\u539f\u59cb\u503c\u4e3a8\uff0c\u8be5\u503c\u5728\u5904\u7406\u5927\u6570\u91cf\u65f6\u53ef\u8bbe\u7f6e\u4e3a\u5f53\u524d\u8282\u70b9\u603b\u7684CPU\u4e2a\u6570\u76841~2\u500d\uff09\u3002<\/p>\n<p>\u201cyarn.nodemanager.resource.memory-mb\u201d\uff1a\u8be5\u53c2\u6570\u6700\u597d\u5bf9\u6bcf\u4e2anodemanaegr\u5206\u522b\u914d\u7f6e\uff0c\u770b\u4e00\u4e0b\u4e3b\u673a\u754c\u9762\u6bcf\u4e2a\u8282\u70b9\u7684\u5185\u5b58\u4f7f\u7528\u7387\uff0c\u6bcf\u4e2anodemanager\u8be5\u53c2\u6570\u914d\u7f6e\u4e3a\uff1a\u7a7a\u95f2\u7684\u5185\u5b58\u6570\u51cf\u53bb8G\uff0c\u6216\u8005\u4e3a\u603b\u5185\u5b58\u768475%\u3002<\/p>\n<ul>\n<li>Solr\u4f7f\u7528\u201cHDFSDirectoryFactory\u201d\u7d22\u5f15\u65b9\u5f0f\u8bfb\u5199HDFS\u4e0a\u7684\u7d22\u5f15\u6587\u4ef6\u65f6\uff0c\u4eceSolr\u4f5c\u4e3aHDFS\u7684Client\u65b9\u9762\u7684\u914d\u7f6e\u8fdb\u884c\u8c03\u4f18\u3002<\/li>\n<\/ul>\n<p>\u4eceSolr\u4f5c\u4e3aHDFS\u7684Client\u65b9\u9762\u7684\u914d\u7f6e\u8fdb\u884c\u8c03\u4f18\u65f6\uff0c\u5982\u679cSolr\u548c\u5176\u4ed6\u7ec4\u4ef6\u8981\u90e8\u7f72\u5728\u76f8\u540c\u7684\u8282\u70b9\u4e0a\uff0c\u5efa\u8bae\u6bcf\u4e2a\u8282\u70b9\u4e0a\u53ea\u90e8\u7f72\u4e00\u4e2aSolr\u5b9e\u4f8b\u3002<\/p>\n<ul>\n<li>\u8c03\u6574HDFS\u7f13\u5b58\u63d0\u9ad8\u7d22\u5f15\u6027\u80fd\u3002Solr\u4e2dHDFS\u76f8\u5173\u7f13\u5b58\uff0c\u901a\u5e38\u5206\u914d\u7cfb\u7edf\u4e0a\u53ef\u7528\u5185\u5b58\u91cf\u768410-20%\u3002\u4f8b\u5982\uff0c\u5728\u6709128GB\u5185\u5b58\u7684\u4e3b\u673a\u4e0a\u8fd0\u884cHDFS\u548cSolr\u65f6\uff0c\u901a\u5e38\u5206\u914d12.8GB\uff5e25.6GB\u7684\u5185\u5b58\u4f5c\u4e3aHDFS\u76f8\u5173\u7f13\u5b58\u3002\u968f\u7740\u7d22\u5f15\u5927\u5c0f\u7684\u589e\u52a0\uff0c\u9700\u8981\u8c03\u6574\u6b64\u53c2\u6570\u4ee5\u4fdd\u6301\u6700\u4f73\u6027\u80fd\u3002\u8be5\u53c2\u6570\u53ef\u901a\u8fc7\u4fee\u6539\u201csolrconfig.xml\u201d\u6587\u4ef6\u8fdb\u884c\u914d\u7f6e\u3002<\/li>\n<\/ul>\n<p>\u8bf7\u6309\u7167\u4ee5\u4e0b\u6b65\u9aa4\uff0c\u8c03\u6574\u5206\u914d\u7684\u7f13\u5b58\u5927\u5c0f\uff1a<\/p>\n<ol>\n<li>\u4f7f\u7528PuTTY\u5de5\u5177\uff0c\u4ee5root\u7528\u6237\u767b\u5f55Solr\u5ba2\u6237\u7aef\u6240\u5728\u8282\u70b9\uff0c\u8fdb\u5165Solr\u5ba2\u6237\u7aef\u6240\u5728\u76ee\u5f55\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff1a<\/li>\n<\/ol>\n<p><strong>source bigdata_env<\/strong><\/p>\n<p><strong>kinit solr<\/strong><\/p>\n<ol>\n<li>\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u83b7\u53d6\u914d\u7f6e\u6587\u4ef6\u96c6\uff0c\u5e76\u6253\u5f00\u201csolrconfig.xml\u201d\u6587\u4ef6\uff1a<\/li>\n<\/ol>\n<p><strong>solrctl confset &#8211;get confWithSchema \/home\/solr\/<\/strong><\/p>\n<p><strong>vi \/home\/solr\/conf\/solrconfig.xml<\/strong><\/p>\n<ol>\n<li>\u4fee\u6539\u53c2\u6570\u201csolr.hdfs.blockcache.slab.count\u201d\u7684\u503c\u201c<!-- -->&lt;int name=&quot;solr.hdfs.blockcache.slab.count&quot;&gt;<!-- -->${solr.hdfs.blockcache.slab.count:1}<!-- -->&lt;\/int&gt;<!-- -->\u201d\u3002<\/li>\n<\/ol>\n<p>\u5176\u4e2d\u6bcf\u4e2aslab\u7684\u5927\u5c0f\u4e3a128MB \uff0c18\u4e2aslab\uff0c\u5927\u7ea6\u5360\u75282.3GB\u5185\u5b58\uff0c\u8fd9\u662f\u6bcf\u4e2a\u5206\u7247Shard\u7684\u914d\u7f6e\uff0c\u5982\u679c\u4e00\u4e2a\u53f0\u4e3b\u673a6\u4e2ashard\uff0c\u90a3\u4e48\u5171\u8ba1\u5360\u752813.8GB\u5185\u5b58\u3002<\/p>\n<p>\u7531\u6b64\u53ef\u77e5\uff0c\u5728\u6b64\u4f8b\u4e2d\u53ef\u5c06\u6539\u53c2\u6570\u503c\u4fee\u6539\u4e3a\u201c<!-- -->&lt;int name=&quot;solr.hdfs.blockcache.slab.count&quot;&gt;<!-- -->${solr.hdfs.blockcache.slab.count:18}<!-- -->&lt;\/int&gt;<!-- -->\u201d\u3002<\/p>\n<ol>\n<li>\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4e0a\u4f20\u4fee\u6539\u8fc7\u7684\u914d\u7f6e\u6587\u4ef6\u96c6\uff1a<\/li>\n<\/ol>\n<p><strong>solrctl confset &#8211;update confWithSchema \/home\/solr\/<\/strong><\/p>\n<h2>12.9\u00a0Spark<\/h2>\n<h3>12.9.1\u00a0Spark Core\u8c03\u4f18<\/h3>\n<p>12.9.1.1\u00a0\u6570\u636e\u5e8f\u5217\u5316<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>Spark\u652f\u6301\u4e24\u79cd\u65b9\u5f0f\u7684\u5e8f\u5217\u5316 \uff1a<\/p>\n<ul>\n<li>Java\u539f\u751f\u5e8f\u5217\u5316JavaSerializer<\/li>\n<li>Kryo\u5e8f\u5217\u5316KryoSerializer<\/li>\n<\/ul>\n<p>\u5e8f\u5217\u5316\u5bf9\u4e8eSpark\u5e94\u7528\u7684\u6027\u80fd\u6765\u8bf4\uff0c\u5177\u6709\u5f88\u5927\u7684\u5f71\u54cd\u3002\u5728\u7279\u5b9a\u7684\u6570\u636e\u683c\u5f0f\u7684\u60c5\u51b5\u4e0b\uff0cKryoSerializer\u7684\u6027\u80fd\u53ef\u4ee5\u8fbe\u5230JavaSerializer\u768410\u500d\u4ee5\u4e0a\uff0c\u800c\u5bf9\u4e8e\u4e00\u4e9bInt\u4e4b\u7c7b\u7684\u57fa\u672c\u7c7b\u578b\u6570\u636e\uff0c\u6027\u80fd\u7684\u63d0\u5347\u5c31\u51e0\u4e4e\u53ef\u4ee5\u5ffd\u7565\u3002<\/p>\n<p>KryoSerializer\u4f9d\u8d56Twitter\u7684Chill\u5e93\u6765\u5b9e\u73b0\uff0c\u76f8\u5bf9\u4e8eJavaSerializer\uff0c\u4e3b\u8981\u7684\u95ee\u9898\u5728\u4e8e\u4e0d\u662f\u6240\u6709\u7684Java Serializable\u5bf9\u8c61\u90fd\u80fd\u652f\u6301\uff0c\u517c\u5bb9\u6027\u4e0d\u597d\uff0c\u6240\u4ee5\u9700\u8981\u624b\u52a8\u6ce8\u518c\u7c7b\u3002<\/p>\n<p>\u5e8f\u5217\u5316\u529f\u80fd\u7528\u5728\u4e24\u4e2a\u5730\u65b9\uff1a\u5e8f\u5217\u5316\u4efb\u52a1\u548c\u5e8f\u5217\u5316\u6570\u636e\u3002Spark\u4efb\u52a1\u5e8f\u5217\u5316\u53ea\u652f\u6301JavaSerializer\uff0c\u6570\u636e\u5e8f\u5217\u5316\u652f\u6301JavaSerializer\u548cKryoSerializer\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>Spark\u7a0b\u5e8f\u8fd0\u884c\u65f6\uff0c\u5728shuffle\u548cRDD Cache\u7b49\u8fc7\u7a0b\u4e2d\uff0c\u4f1a\u6709\u5927\u91cf\u7684\u6570\u636e\u9700\u8981\u5e8f\u5217\u5316\uff0c\u9ed8\u8ba4\u4f7f\u7528JavaSerializer\uff0c\u901a\u8fc7\u914d\u7f6e\u8ba9KryoSerializer\u4f5c\u4e3a\u6570\u636e\u5e8f\u5217\u5316\u5668\u6765\u63d0\u5347\u5e8f\u5217\u5316\u6027\u80fd\u3002<\/p>\n<p>\u5728\u5f00\u53d1\u5e94\u7528\u7a0b\u5e8f\u65f6\uff0c\u6dfb\u52a0\u5982\u4e0b\u4ee3\u7801\u6765\u4f7f\u7528KryoSerializer\u4f5c\u4e3a\u6570\u636e\u5e8f\u5217\u5316\u5668\u3002<\/p>\n<ul>\n<li>\u5b9e\u73b0\u7c7b\u6ce8\u518c\u5668\u5e76\u624b\u52a8\u6ce8\u518c\u7c7b\u3002<\/li>\n<\/ul>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/>6<br \/>7<br \/>8<br \/>9<br \/>10<br \/>11<br \/>12<br \/>13<br \/>14<br \/>15<br \/>16<br \/>17<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2package com.etl.common;<br \/>\n3<br \/>\n4import com.esotericsoftware.kryo.Kryo;<br \/>\n5import org.apache.spark.serializer.KryoRegistrator; <br \/>\n6<br \/>\n7public class DemoRegistrator implements KryoRegistrator<br \/>\n8{<br \/>\n9 &nbsp; &nbsp;@Override<br \/>\n10 &nbsp; &nbsp;public void registerClasses(Kryo kryo)<br \/>\n11 &nbsp; &nbsp;{<br \/>\n12 &nbsp; &nbsp; &nbsp; &nbsp;\/\/\u4ee5\u4e0b\u4e3a\u793a\u4f8b\u7c7b\uff0c\u8bf7\u6ce8\u518c\u81ea\u5b9a\u4e49\u7684\u7c7b<br \/>\n13 &nbsp; &nbsp; &nbsp; &nbsp;kryo.register(AggrateKey.class);<br \/>\n14 &nbsp; &nbsp; &nbsp; &nbsp;kryo.register(AggrateValue.class);<br \/>\n15 &nbsp; &nbsp;}<br \/>\n16}<br \/>\n17<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u60a8\u53ef\u4ee5\u5728Spark\u5ba2\u6237\u7aef\u5bf9spark.kryo.registrationRequired\u53c2\u6570\u8fdb\u884c\u914d\u7f6e\uff0c\u8bbe\u7f6e\u662f\u5426\u9700\u8981Kryo\u6ce8\u518c\u5e8f\u5217\u5316\u3002<\/p>\n<p>\u5f53\u53c2\u6570\u8bbe\u7f6e\u4e3atrue\u65f6\uff0c\u5982\u679c\u5de5\u7a0b\u4e2d\u5b58\u5728\u672a\u88ab\u5e8f\u5217\u5316\u7684\u7c7b\uff0c\u5219\u4f1a\u629b\u51fa\u5f02\u5e38\u3002\u5982\u679c\u8bbe\u7f6e\u4e3afalse\uff08\u9ed8\u8ba4\u503c\uff09\uff0cKryo\u4f1a\u81ea\u52a8\u5c06\u672a\u6ce8\u518c\u7684\u7c7b\u540d\u5199\u5230\u5bf9\u5e94\u7684\u5bf9\u8c61\u4e2d\u3002\u6b64\u64cd\u4f5c\u4f1a\u5bf9\u7cfb\u7edf\u6027\u80fd\u9020\u6210\u5f71\u54cd\u3002\u8bbe\u7f6e\u4e3atrue\u65f6\uff0c\u7528\u6237\u9700\u624b\u52a8\u6ce8\u518c\u7c7b\uff0c\u9488\u5bf9\u672a\u5e8f\u5217\u5316\u7684\u7c7b\uff0c\u7cfb\u7edf\u4e0d\u4f1a\u81ea\u52a8\u5199\u5165\u7c7b\u540d\uff0c\u800c\u662f\u629b\u51fa\u5f02\u5e38\uff0c\u76f8\u5bf9\u6bd4false\uff0c\u5176\u6027\u80fd\u8f83\u597d\u3002<\/p>\n<ul>\n<li>\u914d\u7f6eKryoSerializer\u4f5c\u4e3a\u6570\u636e\u5e8f\u5217\u5316\u5668\u548c\u7c7b\u6ce8\u518c\u5668\u3002<\/li>\n<\/ul>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2val conf = new SparkConf()<br \/>\n3conf.set(&amp;quot;spark.serializer&amp;quot;, &amp;quot;org.apache.spark.serializer.KryoSerializer&amp;quot;)<br \/>\n4.set(&amp;quot;spark.kryo.registrator&amp;quot;, &amp;quot;com.etl.common.DemoRegistrator&amp;quot;)<br \/>\n5<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>12.9.1.2\u00a0\u914d\u7f6e\u5185\u5b58<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>Spark\u662f\u5185\u5b58\u8ba1\u7b97\u6846\u67b6\uff0c\u8ba1\u7b97\u8fc7\u7a0b\u4e2d\u5185\u5b58\u4e0d\u591f\u5bf9Spark\u7684\u6267\u884c\u6548\u7387\u5f71\u54cd\u5f88\u5927\u3002\u53ef\u4ee5\u901a\u8fc7\u76d1\u63a7GC\uff08Garbage Collection\uff09\uff0c\u8bc4\u4f30\u5185\u5b58\u4e2dRDD\u7684\u5927\u5c0f\u6765\u5224\u65ad\u5185\u5b58\u662f\u5426\u53d8\u6210\u6027\u80fd\u74f6\u9888\uff0c\u5e76\u6839\u636e\u60c5\u51b5\u4f18\u5316\u3002<\/p>\n<p>\u76d1\u63a7\u8282\u70b9\u8fdb\u7a0b\u7684GC\u60c5\u51b5\uff08\u5728\u5ba2\u6237\u7aef\u7684conf\/spark-default.conf\u914d\u7f6e\u6587\u4ef6\u4e2d\uff0c\u5728spark.driver.extraJavaOptions\u548cspark.executor.extraJavaOptions\u914d\u7f6e\u9879\u4e2d\u6dfb\u52a0\u53c2\u6570\uff1a&quot;-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps&quot;<\/p>\n<p>\uff09\uff0c\u5982\u679c\u9891\u7e41\u51fa\u73b0Full GC\uff0c\u9700\u8981\u4f18\u5316GC\u3002\u628aRDD\u505aCache\u64cd\u4f5c\uff0c\u901a\u8fc7\u65e5\u5fd7\u67e5\u770bRDD\u5728\u5185\u5b58\u4e2d\u7684\u5927\u5c0f\uff0c\u5982\u679c\u6570\u636e\u592a\u5927\uff0c\u9700\u8981\u6539\u53d8RDD\u7684\u5b58\u50a8\u7ea7\u522b\u6765\u4f18\u5316\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ul>\n<li>\n<p>\u4f18\u5316GC\uff0c\u8c03\u6574\u8001\u5e74\u4ee3\u548c\u65b0\u751f\u4ee3\u7684\u5927\u5c0f\u548c\u6bd4\u4f8b\u3002\u5728\u5ba2\u6237\u7aef\u7684conf\/spark-default.conf\u914d\u7f6e\u6587\u4ef6\u4e2d\uff0c\u5728spark.driver.extraJavaOptions\u548cspark.executor.extraJavaOptions\u914d\u7f6e\u9879\u4e2d\u6dfb\u52a0\u53c2\u6570\uff1a-XX:NewRatio\u3002\u5982\uff0c&quot; -XX:NewRatio=2&quot;\uff0c\u5219\u65b0\u751f\u4ee3\u5360\u6574\u4e2a\u5806\u7a7a\u95f4\u76841\/3\uff0c\u8001\u5e74\u4ee3\u53602\/3\u3002<\/p>\n<\/li>\n<li>\n<p>\u5f00\u53d1Spark\u5e94\u7528\u7a0b\u5e8f\u65f6\uff0c\u4f18\u5316RDD\u7684\u6570\u636e\u7ed3\u6784\u3002<\/p>\n<\/li>\n<li>\n<p>\u4f7f\u7528\u539f\u59cb\u7c7b\u578b\u6570\u7ec4\u66ff\u4ee3\u96c6\u5408\u7c7b\uff0c\u5982\u53ef\u4f7f\u7528fastutil\u5e93\u3002<\/p>\n<ul>\n<li>\u907f\u514d\u5d4c\u5957\u7ed3\u6784\u3002<\/li>\n<li>Key\u5c3d\u91cf\u4e0d\u8981\u4f7f\u7528String\u3002<\/li>\n<\/ul>\n<\/li>\n<li>\n<p>\u5f00\u53d1Spark\u5e94\u7528\u7a0b\u5e8f\u65f6\uff0c\u5efa\u8bae\u5e8f\u5217\u5316RDD\u3002<\/p>\n<\/li>\n<\/ul>\n<p>RDD\u505acache\u65f6\u9ed8\u8ba4\u662f\u4e0d\u5e8f\u5217\u5316\u6570\u636e\u7684\uff0c\u53ef\u4ee5\u901a\u8fc7\u8bbe\u7f6e\u5b58\u50a8\u7ea7\u522b\u6765\u5e8f\u5217\u5316RDD\u51cf\u5c0f\u5185\u5b58\u3002\u4f8b\u5982\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2testRDD.persist(StorageLevel.MEMORY_ONLY_SER)<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>12.9.1.3\u00a0\u8bbe\u7f6e\u5e76\u884c\u5ea6<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5e76\u884c\u5ea6\u63a7\u5236\u4efb\u52a1\u7684\u6570\u91cf\uff0c\u5f71\u54cdshuffle\u64cd\u4f5c\u540e\u6570\u636e\u88ab\u5207\u5206\u6210\u7684\u5757\u6570\u3002\u8c03\u6574\u5e76\u884c\u5ea6\u8ba9\u4efb\u52a1\u7684\u6570\u91cf\u548c\u6bcf\u4e2a\u4efb\u52a1\u5904\u7406\u7684\u6570\u636e\u4e0e\u673a\u5668\u7684\u5904\u7406\u80fd\u529b\u8fbe\u5230\u6700\u4f18\u3002<\/p>\n<p>\u67e5\u770bCPU\u4f7f\u7528\u60c5\u51b5\u548c\u5185\u5b58\u5360\u7528\u60c5\u51b5\uff0c\u5f53\u4efb\u52a1\u548c\u6570\u636e\u4e0d\u662f\u5e73\u5747\u5206\u5e03\u5728\u5404\u8282\u70b9\uff0c\u800c\u662f\u96c6\u4e2d\u5728\u4e2a\u522b\u8282\u70b9\u65f6\uff0c\u53ef\u4ee5\u589e\u5927\u5e76\u884c\u5ea6\u4f7f\u4efb\u52a1\u548c\u6570\u636e\u66f4\u5747\u5300\u7684\u5206\u5e03\u5728\u5404\u4e2a\u8282\u70b9\u3002\u589e\u52a0\u4efb\u52a1\u7684\u5e76\u884c\u5ea6\uff0c\u5145\u5206\u5229\u7528\u96c6\u7fa4\u673a\u5668\u7684\u8ba1\u7b97\u80fd\u529b\uff0c\u4e00\u822c\u5e76\u884c\u5ea6\u8bbe\u7f6e\u4e3a\u96c6\u7fa4CPU\u603b\u548c\u76842-3\u500d\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u5e76\u884c\u5ea6\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u4e09\u79cd\u65b9\u5f0f\u6765\u8bbe\u7f6e\uff0c\u7528\u6237\u53ef\u4ee5\u6839\u636e\u5b9e\u9645\u7684\u5185\u5b58\u3001CPU\u3001\u6570\u636e\u4ee5\u53ca\u5e94\u7528\u7a0b\u5e8f\u903b\u8f91\u7684\u60c5\u51b5\u8c03\u6574\u5e76\u884c\u5ea6\u53c2\u6570\u3002<\/p>\n<ul>\n<li>\u5728\u4f1a\u4ea7\u751fshuffle\u7684\u64cd\u4f5c\u51fd\u6570\u5185\u8bbe\u7f6e\u5e76\u884c\u5ea6\u53c2\u6570\uff0c\u4f18\u5148\u7ea7\u6700\u9ad8\u3002<\/li>\n<\/ul>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2testRDD.groupByKey(24)<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<ul>\n<li>\u5728\u4ee3\u7801\u4e2d\u914d\u7f6e\u201cspark.default.parallelism\u201d\u8bbe\u7f6e\u5e76\u884c\u5ea6\uff0c\u4f18\u5148\u7ea7\u6b21\u4e4b\u3002<\/li>\n<\/ul>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2val conf = new SparkConf()<br \/>\n3conf.set(&amp;quot;spark.default.parallelism&amp;quot;, 24)<br \/>\n4<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<ul>\n<li>\u5728\u201c$SPARK_HOME\/conf\/spark-defaults.conf\u201d\u6587\u4ef6\u4e2d\u914d\u7f6e\u201cspark.default.parallelism\u201d\u7684\u503c\uff0c\u4f18\u5148\u7ea7\u6700\u4f4e\u3002<\/li>\n<\/ul>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2spark.default.parallelism &nbsp; &nbsp;24<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>12.9.1.4\u00a0\u4f7f\u7528\u5e7f\u64ad\u53d8\u91cf<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>Broadcast\uff08\u5e7f\u64ad\uff09\u53ef\u4ee5\u628a\u6570\u636e\u96c6\u5408\u5206\u53d1\u5230\u6bcf\u4e00\u4e2a\u8282\u70b9\u4e0a\uff0cSpark\u4efb\u52a1\u5728\u6267\u884c\u8fc7\u7a0b\u4e2d\u8981\u4f7f\u7528\u8fd9\u4e2a\u6570\u636e\u96c6\u5408\u65f6\uff0c\u5c31\u4f1a\u5728\u672c\u5730\u67e5\u627eBroadcast\u8fc7\u6765\u7684\u6570\u636e\u96c6\u5408\u3002\u5982\u679c\u4e0d\u4f7f\u7528Broadcast\uff0c\u6bcf\u6b21\u4efb\u52a1\u9700\u8981\u6570\u636e\u96c6\u5408\u65f6\uff0c\u90fd\u4f1a\u628a\u6570\u636e\u5e8f\u5217\u5316\u5230\u4efb\u52a1\u91cc\u9762\uff0c\u4e0d\u4f46\u8017\u65f6\uff0c\u8fd8\u4f7f\u4efb\u52a1\u53d8\u5f97\u5f88\u5927\u3002<\/p>\n<ol>\n<li>\u6bcf\u4e2a\u4efb\u52a1\u5206\u7247\u5728\u6267\u884c\u4e2d\u90fd\u9700\u8981\u540c\u4e00\u4efd\u6570\u636e\u96c6\u5408\u65f6\uff0c\u5c31\u53ef\u4ee5\u628a\u516c\u5171\u6570\u636e\u96c6Broadcast\u5230\u6bcf\u4e2a\u8282\u70b9\uff0c\u8ba9\u6bcf\u4e2a\u8282\u70b9\u5728\u672c\u5730\u90fd\u4fdd\u5b58\u4e00\u4efd\u3002<\/li>\n<li>\u5927\u8868\u548c\u5c0f\u8868\u505ajoin\u64cd\u4f5c\u65f6\u53ef\u4ee5\u628a\u5c0f\u8868Broadcast\u5230\u5404\u4e2a\u8282\u70b9\uff0c\u4ece\u800c\u5c31\u53ef\u4ee5\u628ajoin\u64cd\u4f5c\u8f6c\u53d8\u6210\u666e\u901a\u7684\u64cd\u4f5c\uff0c\u51cf\u5c11\u4e86shuffle\u64cd\u4f5c\u3002<\/li>\n<\/ol>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u5728\u5f00\u53d1\u5e94\u7528\u7a0b\u5e8f\u65f6\uff0c\u6dfb\u52a0\u5982\u4e0b\u4ee3\u7801\uff0c\u5c06\u201ctestArr\u201d\u6570\u636e\u5e7f\u64ad\u5230\u5404\u4e2a\u8282\u70b9\u3002<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/>6<br \/>7<br \/>8<br \/>9<br \/>10<br \/>11<br \/>12<br \/>13<br \/>14<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2def main(args: Array[String]) {<br \/>\n3 &nbsp;...<br \/>\n4 &nbsp;val testArr: Array[Long] = new Array[Long](200)<br \/>\n5 &nbsp;val testBroadcast: Broadcast[Array[Long]] = sc.broadcast(testArr)<br \/>\n6 &nbsp;val resultRdd: RDD[Long] = inpputRdd.map(input =&amp;gt; handleData(testBroadcast, input))<br \/>\n7 &nbsp;...<br \/>\n8}<br \/>\n9<br \/>\n10def handleData(broadcast: Broadcast[Array[Long]], input: String) {<br \/>\n11 &nbsp;val value = broadcast.value<br \/>\n12 &nbsp;...<br \/>\n13}<br \/>\n14<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>12.9.1.5\u00a0\u4f7f\u7528External Shuffle Service\u63d0\u5347\u6027\u80fd<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>Spark\u7cfb\u7edf\u5728\u8fd0\u884c\u542bshuffle\u8fc7\u7a0b\u7684\u5e94\u7528\u65f6\uff0cExecutor\u8fdb\u7a0b\u9664\u4e86\u8fd0\u884ctask\uff0c\u8fd8\u8981\u8d1f\u8d23\u5199shuffle\u6570\u636e\u4ee5\u53ca\u7ed9\u5176\u4ed6Executor\u63d0\u4f9bshuffle\u6570\u636e\u3002\u5f53Executor\u8fdb\u7a0b\u4efb\u52a1\u8fc7\u91cd\uff0c\u5bfc\u81f4\u89e6\u53d1GC\uff08Garbage Collection\uff09\u800c\u4e0d\u80fd\u4e3a\u5176\u4ed6Executor\u63d0\u4f9bshuffle\u6570\u636e\u65f6\uff0c\u4f1a\u5f71\u54cd\u4efb\u52a1\u8fd0\u884c\u3002<\/p>\n<p>External shuffle Service\u662f\u957f\u671f\u5b58\u5728\u4e8eNodeManager\u8fdb\u7a0b\u4e2d\u7684\u4e00\u4e2a\u8f85\u52a9\u670d\u52a1\u3002\u901a\u8fc7\u8be5\u670d\u52a1\u6765\u6293\u53d6shuffle\u6570\u636e\uff0c\u51cf\u5c11\u4e86Executor\u7684\u538b\u529b\uff0c\u5728Executor GC\u7684\u65f6\u5019\u4e5f\u4e0d\u4f1a\u5f71\u54cd\u5176\u4ed6Executor\u7684\u4efb\u52a1\u8fd0\u884c\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ol>\n<li>\u767b\u5f55FusionInsight Manager\u7cfb\u7edf\u3002<\/li>\n<li>\u5355\u51fb\u201c\u670d\u52a1\u7ba1\u7406 &gt; Spark &gt; \u670d\u52a1\u914d\u7f6e\u201d\u3002\u5728\u201c\u53c2\u6570\u7c7b\u522b\u201d\u4e2d\u9009\u62e9\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002<\/li>\n<li>\u9009\u62e9\u201cSparkResource &gt; \u9ed8\u8ba4\u201d\uff0c\u4fee\u6539\u4ee5\u4e0b\u53c2\u6570\uff1a\u886812-23\u00a0\u53c2\u6570\u5217\u8868<\/li>\n<\/ol>\n<p>spark.shuffle.service.enabled<br \/>\nfalse<br \/>\ntrue<\/p>\n<ol>\n<li>\u91cd\u542fSpark\u670d\u52a1\uff0c\u914d\u7f6e\u751f\u6548\u3002\u00a0\u8bf4\u660e\uff1a<\/li>\n<\/ol>\n<p>\u5982\u679c\u9700\u8981\u5728Spark\u5ba2\u6237\u7aef\u7528External Shuffle Service\u529f\u80fd\uff0c\u9700\u8981\u91cd\u65b0\u4e0b\u8f7d\u5e76\u5b89\u88c5Spark\u5ba2\u6237\u7aef\uff0c\u5177\u4f53\u64cd\u4f5c\u8bf7\u53c2\u89c1\u300a\u5b89\u88c5\u5ba2\u6237\u7aef\u300b\u7ae0\u8282\u3002<\/p>\n<p>12.9.1.6\u00a0Yarn\u6a21\u5f0f\u4e0b\u52a8\u6001\u8d44\u6e90\u8c03\u5ea6<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5bf9\u4e8eSpark\u5e94\u7528\u6765\u8bf4\uff0c\u8d44\u6e90\u662f\u5f71\u54cdSpark\u5e94\u7528\u6267\u884c\u6548\u7387\u7684\u4e00\u4e2a\u91cd\u8981\u56e0\u7d20\u3002\u5f53\u4e00\u4e2a\u957f\u671f\u8fd0\u884c\u7684\u670d\u52a1\uff08\u6bd4\u5982JDBCServer\uff09\uff0c\u82e5\u5206\u914d\u7ed9\u5b83\u591a\u4e2aExecutor\uff0c\u53ef\u662f\u5374\u6ca1\u6709\u4efb\u4f55\u4efb\u52a1\u5206\u914d\u7ed9\u5b83\uff0c\u800c\u6b64\u65f6\u6709\u5176\u4ed6\u7684\u5e94\u7528\u5374\u8d44\u6e90\u7d27\u5f20\uff0c\u8fd9\u5c31\u9020\u6210\u4e86\u5f88\u5927\u7684\u8d44\u6e90\u6d6a\u8d39\u548c\u8d44\u6e90\u4e0d\u5408\u7406\u7684\u8c03\u5ea6\u3002<\/p>\n<p>\u52a8\u6001\u8d44\u6e90\u8c03\u5ea6\u5c31\u662f\u4e3a\u4e86\u89e3\u51b3\u8fd9\u79cd\u573a\u666f\uff0c\u6839\u636e\u5f53\u524d\u5e94\u7528\u4efb\u52a1\u7684\u8d1f\u8f7d\u60c5\u51b5\uff0c\u5b9e\u65f6\u7684\u589e\u51cfExecutor\u4e2a\u6570\uff0c\u4ece\u800c\u5b9e\u73b0\u52a8\u6001\u5206\u914d\u8d44\u6e90\uff0c\u4f7f\u6574\u4e2aSpark\u7cfb\u7edf\u66f4\u52a0\u5065\u5eb7\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ol>\n<li>\u9700\u8981\u5148\u914d\u7f6eExternal shuffle service\u3002<\/li>\n<li>\u767b\u5f55FusionInsight Manager\uff0c\u5c06\u201cspark.dynamicAllocation.enabled\u201d\u53c2\u6570\u7684\u503c\u8bbe\u7f6e\u4e3a\u201ctrue\u201d\uff0c\u8868\u793a\u5f00\u542f\u52a8\u6001\u8d44\u6e90\u8c03\u5ea6\u529f\u80fd\u3002\u9ed8\u8ba4\u60c5\u51b5\u4e0b\u5173\u95ed\u6b64\u529f\u80fd\u3002<\/li>\n<li>\u4e0b\u9762\u662f\u4e00\u4e9b\u53ef\u9009\u914d\u7f6e\uff0c\u5982\u886812-24\u6240\u793a\u3002<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p>\u886812-24\u00a0\u52a8\u6001\u8d44\u6e90\u8c03\u5ea6\u53c2\u6570<\/p>\n<p>spark.dynamicAllocation.minExecutors<br \/>\n\u6700\u5c0fExecutor\u4e2a\u6570\u3002<br \/>\n0<br \/>\nspark.dynamicAllocation.initialExecutors<br \/>\n\u521d\u59cbExecutor\u4e2a\u6570\u3002<br \/>\nspark.dynamicAllocation.minExecutors<br \/>\nspark.dynamicAllocation.maxExecutors<br \/>\n\u6700\u5927Executor\u4e2a\u6570\u3002<br \/>\n2048<br \/>\nspark.dynamicAllocation.schedulerBacklogTimeout<br \/>\n\u8c03\u5ea6\u7b2c\u4e00\u6b21\u8d85\u65f6\u65f6\u95f4\u3002<br \/>\n1(s)<br \/>\nspark.dynamicAllocation.sustainedSchedulerBacklogTimeout<br \/>\n\u8c03\u5ea6\u7b2c\u4e8c\u6b21\u53ca\u4e4b\u540e\u8d85\u65f6\u65f6\u95f4\u3002<br \/>\nspark.dynamicAllocation.schedulerBacklogTimeout<br \/>\nspark.dynamicAllocation.executorIdleTimeout<br \/>\n\u666e\u901aExecutor\u7a7a\u95f2\u8d85\u65f6\u65f6\u95f4\u3002<br \/>\n60(s)<br \/>\nspark.dynamicAllocation.cachedExecutorIdleTimeout<br \/>\n\u542b\u6709cached blocks\u7684Executor\u7a7a\u95f2\u8d85\u65f6\u65f6\u95f4\u3002<br \/>\nInteger.MAX_VALUE\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u4f7f\u7528\u52a8\u6001\u8d44\u6e90\u8c03\u5ea6\u529f\u80fd\uff0c\u5fc5\u987b\u914d\u7f6eExternal Shuffle Service\u3002<\/p>\n<p>\u00a0<\/p>\n<p>12.9.1.7\u00a0\u914d\u7f6e\u8fdb\u7a0b\u53c2\u6570<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>Spark on YARN\u6a21\u5f0f\u4e0b\uff0c\u6709Driver\u3001ApplicationMaster\u3001Executor\u4e09\u79cd\u8fdb\u7a0b\u3002\u5728\u4efb\u52a1\u8c03\u5ea6\u548c\u8fd0\u884c\u7684\u8fc7\u7a0b\u4e2d\uff0cDriver\u548cExecutor\u627f\u62c5\u4e86\u5f88\u5927\u7684\u8d23\u4efb\uff0c\u800cApplicationMaster\u4e3b\u8981\u8d1f\u8d23container\u7684\u542f\u505c\u3002<\/p>\n<p>\u56e0\u800cDriver\u548cExecutor\u7684\u53c2\u6570\u914d\u7f6e\u5bf9spark\u5e94\u7528\u7684\u6267\u884c\u6709\u7740\u5f88\u5927\u7684\u5f71\u54cd\u610f\u4e49\u3002\u7528\u6237\u53ef\u901a\u8fc7\u5982\u4e0b\u64cd\u4f5c\u5bf9Spark\u96c6\u7fa4\u6027\u80fd\u505a\u4f18\u5316\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ol>\n<li>\u914d\u7f6eDriver\u5185\u5b58\u3002<\/li>\n<\/ol>\n<p>\u00a0<\/p>\n<p>Driver\u8d1f\u8d23\u4efb\u52a1\u7684\u8c03\u5ea6\uff0c\u548cExecutor\u3001AM\u4e4b\u95f4\u7684\u6d88\u606f\u901a\u4fe1\u3002\u5f53\u4efb\u52a1\u6570\u53d8\u591a\uff0c\u4efb\u52a1\u5e73\u884c\u5ea6\u589e\u5927\u65f6\uff0cDriver\u5185\u5b58\u90fd\u9700\u8981\u76f8\u5e94\u589e\u5927\u3002<\/p>\n<p>\u60a8\u53ef\u4ee5\u6839\u636e\u5b9e\u9645\u4efb\u52a1\u6570\u91cf\u7684\u591a\u5c11\uff0c\u4e3aDriver\u8bbe\u7f6e\u4e00\u4e2a\u5408\u9002\u7684\u5185\u5b58\u3002<\/p>\n<ul>\n<li>\u5c06\u201cspark-defaults.conf\u201d\u4e2d\u7684\u201cspark.driver.memory\u201d\u914d\u7f6e\u9879\u6216\u8005\u201cspark-env.sh\u201d\u4e2d\u7684\u201cSPARK_DRIVER_MEMORY\u201d\u914d\u7f6e\u9879\u8bbe\u7f6e\u4e3a\u5408\u9002\u5927\u5c0f\u3002<!-- -->\n<ul>\n<li>\u5728\u4f7f\u7528spark-submit\u547d\u4ee4\u65f6\uff0c\u6dfb\u52a0\u201c&#8211;driver-memory MEM\u201d\u53c2\u6570\u8bbe\u7f6e\u5185\u5b58\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u00a0<\/p>\n<ul>\n<li>\u914d\u7f6eExecutor\u4e2a\u6570\u3002<\/li>\n<\/ul>\n<p>\u00a0<\/p>\n<p>\u6bcf\u4e2aExecutor\u6bcf\u4e2a\u6838\u540c\u65f6\u80fd\u8dd1\u4e00\u4e2atask\uff0c\u6240\u4ee5\u589e\u52a0\u4e86Executor\u7684\u4e2a\u6570\u76f8\u5f53\u4e8e\u589e\u5927\u4e86\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u3002\u5728\u8d44\u6e90\u5145\u8db3\u7684\u60c5\u51b5\u4e0b\uff0c\u53ef\u4ee5\u76f8\u5e94\u589e\u52a0Executor\u7684\u4e2a\u6570\uff0c\u4ee5\u63d0\u9ad8\u8fd0\u884c\u6548\u7387\u3002<\/p>\n<ul>\n<li>\u5c06\u201cspark-defaults.conf\u201d\u4e2d\u7684\u201cspark.executor.instance\u201d\u914d\u7f6e\u9879\u6216\u8005\u201cspark-env.sh\u201d\u4e2d\u7684\u201cSPARK_EXECUTOR_INSTANCES\u201d\u914d\u7f6e\u9879\u8bbe\u7f6e\u4e3a\u5408\u9002\u5927\u5c0f\u3002\u60a8\u8fd8\u53ef\u4ee5\u8bbe\u7f6e\u52a8\u6001\u8d44\u6e90\u8c03\u5ea6\u529f\u80fd\u8fdb\u884c\u4f18\u5316\uff0c\u8be6\u60c5\u8bf7\u53c2\u89c1\u201cYarn\u6a21\u5f0f\u4e0b\u52a8\u6001\u8d44\u6e90\u8c03\u5ea6\u201d\u7ae0\u8282\u3002<!-- -->\n<ul>\n<li>\u5728\u4f7f\u7528spark-submit\u547d\u4ee4\u65f6\uff0c\u6dfb\u52a0\u201c&#8211;num-executors NUM\u201d\u53c2\u6570\u8bbe\u7f6eExecutor\u4e2a\u6570\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u00a0<\/p>\n<ul>\n<li>\u914d\u7f6eExecutor\u6838\u6570\u3002<\/li>\n<\/ul>\n<p>\u00a0<\/p>\n<p>\u6bcf\u4e2aExecutor\u591a\u4e2a\u6838\u540c\u65f6\u80fd\u8dd1\u591a\u4e2atask\uff0c\u76f8\u5f53\u4e8e\u589e\u5927\u4e86\u4efb\u52a1\u7684\u5e76\u53d1\u5ea6\u3002\u4f46\u662f\u7531\u4e8e\u6240\u6709\u6838\u5171\u7528Executor\u7684\u5185\u5b58\uff0c\u6240\u4ee5\u8981\u5728\u5185\u5b58\u548c\u6838\u6570\u4e4b\u95f4\u505a\u597d\u5e73\u8861\u3002<\/p>\n<ul>\n<li>\u5c06\u201cspark-defaults.conf\u201d\u4e2d\u7684\u201cspark.executor.cores\u201d\u914d\u7f6e\u9879\u6216\u8005\u201cspark-env.sh\u201d\u4e2d\u7684\u201cSPARK_EXECUTOR_CORES\u201d\u914d\u7f6e\u9879\u8bbe\u7f6e\u4e3a\u5408\u9002\u5927\u5c0f\u3002<!-- -->\n<ul>\n<li>\u5728\u4f7f\u7528spark-submit\u547d\u4ee4\u65f6\uff0c\u6dfb\u52a0\u201c&#8211;executor-cores NUM\u201d\u53c2\u6570\u8bbe\u7f6e\u6838\u6570\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u00a0<\/p>\n<ul>\n<li>\u914d\u7f6eExecutor\u5185\u5b58\u3002<\/li>\n<\/ul>\n<p>\u00a0<\/p>\n<p>Executor\u7684\u5185\u5b58\u4e3b\u8981\u7528\u4e8e\u4efb\u52a1\u6267\u884c\u3001\u901a\u4fe1\u7b49\u3002\u5f53\u4e00\u4e2a\u4efb\u52a1\u5f88\u5927\u7684\u65f6\u5019\uff0c\u53ef\u80fd\u9700\u8981\u8f83\u591a\u8d44\u6e90\uff0c\u56e0\u800c\u5185\u5b58\u4e5f\u53ef\u4ee5\u505a\u76f8\u5e94\u7684\u589e\u52a0\uff1b\u5f53\u4e00\u4e2a\u4efb\u52a1\u8f83\u5c0f\u8fd0\u884c\u8f83\u5feb\u65f6\uff0c\u5c31\u53ef\u4ee5\u589e\u5927\u5e76\u53d1\u5ea6\u51cf\u5c11\u5185\u5b58\u3002<\/p>\n<ul>\n<li>\u5c06\u201cspark-defaults.conf\u201d\u4e2d\u7684\u201cspark.executor.memory\u201d\u914d\u7f6e\u9879\u6216\u8005\u201cspark-env.sh\u201d\u4e2d\u7684\u201cSPARK_EXECUTOR_MEMORY\u201d\u914d\u7f6e\u9879\u8bbe\u7f6e\u4e3a\u5408\u9002\u5927\u5c0f\u3002<!-- -->\n<ul>\n<li>\u5728\u4f7f\u7528spark-submit\u547d\u4ee4\u65f6\uff0c\u6dfb\u52a0\u201c&#8211;executor-memory MEM\u201d\u53c2\u6570\u8bbe\u7f6e\u5185\u5b58\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u00a0<\/p>\n<p>\u793a\u4f8b<\/p>\n<ul>\n<li>\u5728\u6267\u884cspark wordcount\u8ba1\u7b97\u4e2d\u30021.6T\u6570\u636e\uff0c250\u4e2aexecutor\u3002<\/li>\n<\/ul>\n<p>\u5728\u9ed8\u8ba4\u53c2\u6570\u4e0b\u6267\u884c\u5931\u8d25\uff0c\u51fa\u73b0Futures timed out\u548cOOM\u9519\u8bef\u3002<\/p>\n<p>\u56e0\u4e3a\u6570\u636e\u91cf\u5927\uff0ctask\u6570\u591a\uff0c\u800cwordcount\u6bcf\u4e2atask\u90fd\u6bd4\u8f83\u5c0f\uff0c\u5b8c\u6210\u901f\u5ea6\u5feb\u3002\u5f53task\u6570\u591a\u65f6driver\u7aef\u76f8\u5e94\u7684\u4e00\u4e9b\u5bf9\u8c61\u5c31\u53d8\u5927\u4e86\uff0c\u800c\u4e14\u6bcf\u4e2atask\u5b8c\u6210\u65f6executor\u548cdriver\u90fd\u8981\u901a\u4fe1\uff0c\u8fd9\u5c31\u4f1a\u5bfc\u81f4\u7531\u4e8e\u5185\u5b58\u4e0d\u8db3\uff0c\u8fdb\u7a0b\u4e4b\u95f4\u901a\u4fe1\u65ad\u8fde\u7b49\u95ee\u9898\u3002<\/p>\n<p>\u5f53\u628aDriver\u7684\u5185\u5b58\u8bbe\u7f6e\u52304g\u65f6\uff0c\u5e94\u7528\u6210\u529f\u8dd1\u5b8c\u3002<\/p>\n<ul>\n<li>\u4f7f\u7528JDBCServer\u6267\u884cTPC-DS\u6d4b\u8bd5\u5957\uff0c\u9ed8\u8ba4\u53c2\u6570\u914d\u7f6e\u4e0b\u4e5f\u62a5\u4e86\u5f88\u591a\u9519\u8bef\uff1aExecutor Lost\u7b49\u3002\u800c\u5f53\u914d\u7f6eDriver\u5185\u5b58\u4e3a30g\uff0cexecutor\u6838\u6570\u4e3a2\uff0cexecutor\u4e2a\u6570\u4e3a125\uff0cexecutor\u5185\u5b58\u4e3a6g\u65f6\uff0c\u6240\u6709\u4efb\u52a1\u624d\u6267\u884c\u6210\u529f\u3002<\/li>\n<\/ul>\n<p>12.9.1.8\u00a0\u8bbe\u8ba1DAG<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5408\u7406\u7684\u8bbe\u8ba1\u7a0b\u5e8f\u7ed3\u6784\uff0c\u53ef\u4ee5\u4f18\u5316\u6267\u884c\u6548\u7387\u3002\u5728\u7a0b\u5e8f\u7f16\u5199\u8fc7\u7a0b\u4e2d\u8981\u5c3d\u91cf\u51cf\u5c11shuffle\u64cd\u4f5c\uff0c\u5408\u5e76\u7a84\u4f9d\u8d56\u64cd\u4f5c\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u4ee5\u201c\u540c\u884c\u8f66\u5224\u65ad\u201d\u4f8b\u5b50\u8bb2\u89e3DAG\u8bbe\u8ba1\u7684\u601d\u8def\u3002<\/p>\n<ul>\n<li>\n<p>**\u6570\u636e\u683c\u5f0f\uff1a**\u901a\u8fc7\u6536\u8d39\u7ad9\u65f6\u95f4\u3001\u8f66\u724c\u53f7\u3001\u6536\u8d39\u7ad9\u7f16\u53f7&#8230;&#8230;<\/p>\n<\/li>\n<li>\n<p>**\u903b\u8f91\uff1a**\u4ee5\u4e0b\u4e24\u79cd\u60c5\u51b5\u4e0b\u5224\u5b9a\u8fd9\u4e24\u8f86\u8f66\u662f\u540c\u884c\u8f66\uff1a<\/p>\n<\/li>\n<li>\n<p>\u5982\u679c\u4e24\u8f86\u8f66\u90fd\u901a\u8fc7\u76f8\u540c\u5e8f\u5217\u7684\u6536\u8d39\u7ad9\uff0c<\/p>\n<ul>\n<li>\u901a\u8fc7\u540c\u4e00\u6536\u8d39\u7ad9\u4e4b\u95f4\u7684\u65f6\u95f4\u5dee\u5c0f\u4e8e\u4e00\u4e2a\u7279\u5b9a\u7684\u503c\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u8be5\u4f8b\u5b50\u6709\u4e24\u79cd\u5b9e\u73b0\u6a21\u5f0f\uff0c\u5176\u4e2d\u5b9e\u73b01\u7684\u903b\u8f91\u5982\u56fe12-2\u6240\u793a\uff0c\u5b9e\u73b02\u7684\u903b\u8f91\u5982\u56fe12-3\u6240\u793a\u3002<\/p>\n<p>\u56fe12-2\u00a0\u5b9e\u73b01\u903b\u8f91<\/p>\n<p>\u5b9e\u73b01\u7684\u903b\u8f91\u8bf4\u660e \uff1a<\/p>\n<ol>\n<li>\u6839\u636e\u8f66\u724c\u53f7\u805a\u5408\u8be5\u8f66\u901a\u8fc7\u7684\u6240\u6709\u6536\u8d39\u7ad9\u5e76\u6392\u5e8f\uff0c\u5904\u7406\u540e\u6570\u636e\u5982\u4e0b\uff1a<\/li>\n<\/ol>\n<p>\u8f66\u724c\u53f71\uff0c[\uff08\u901a\u8fc7\u65f6\u95f4\uff0c\u6536\u8d39\u7ad93\uff09\uff0c\uff08\u901a\u8fc7\u65f6\u95f4\uff0c\u6536\u8d39\u7ad92\uff09\uff0c\uff08\u901a\u8fc7\u65f6\u95f4\uff0c\u6536\u8d39\u7ad94\uff09\uff0c\uff08\u901a\u8fc7\u65f6\u95f4\uff0c\u6536\u8d39\u7ad95\uff09]<\/p>\n<ol>\n<li>\u6807\u8bc6\u8be5\u6536\u8d39\u7ad9\u662f\u8fd9\u8f86\u8f66\u901a\u8fc7\u7684\u7b2c\u51e0\u4e2a\u6536\u8d39\u7ad9\u3002<\/li>\n<\/ol>\n<p>\uff08\u6536\u8d39\u7ad93\uff0c\uff08\u8f66\u724c\u53f71\uff0c\u901a\u8fc7\u65f6\u95f4\uff0c\u901a\u8fc7\u7684\u7b2c1\u4e2a\u6536\u8d39\u7ad9\uff09\uff09<\/p>\n<p>\uff08\u6536\u8d39\u7ad92\uff0c\uff08\u8f66\u724c\u53f71\uff0c\u901a\u8fc7\u65f6\u95f4\uff0c\u901a\u8fc7\u7684\u7b2c2\u4e2a\u6536\u8d39\u7ad9\uff09\uff09<\/p>\n<p>\uff08\u6536\u8d39\u7ad94\uff0c\uff08\u8f66\u724c\u53f71\uff0c\u901a\u8fc7\u65f6\u95f4\uff0c\u901a\u8fc7\u7684\u7b2c3\u4e2a\u6536\u8d39\u7ad9\uff09\uff09<\/p>\n<p>\uff08\u6536\u8d39\u7ad95\uff0c\uff08\u8f66\u724c\u53f71\uff0c\u901a\u8fc7\u65f6\u95f4\uff0c\u901a\u8fc7\u7684\u7b2c4\u4e2a\u6536\u8d39\u7ad9\uff09\uff09<\/p>\n<ol>\n<li>\u6839\u636e\u6536\u8d39\u7ad9\u805a\u5408\u6570\u636e\u3002<\/li>\n<\/ol>\n<p>\u6536\u8d39\u7ad91\uff0c[\uff08\u8f66\u724c\u53f71\uff0c\u901a\u8fc7\u65f6\u95f4\uff0c\u901a\u8fc7\u7684\u7b2c1\u4e2a\u6536\u8d39\u7ad9\uff09\uff0c\uff08\u8f66\u724c\u53f72\uff0c\u901a\u8fc7\u65f6\u95f4\uff0c\u901a\u8fc7\u7684\u7b2c5\u4e2a\u6536\u8d39\u7ad9\uff09\uff0c\uff08\u8f66\u724c\u53f73\uff0c\u901a\u8fc7\u65f6\u95f4\uff0c\u901a\u8fc7\u7684\u7b2c2\u4e2a\u6536\u8d39\u7ad9\uff09]<\/p>\n<ol>\n<li>\u5224\u65ad\u4e24\u8f86\u8f66\u901a\u8fc7\u8be5\u6536\u8d39\u7ad9\u7684\u65f6\u95f4\u5dee\u662f\u5426\u6ee1\u8db3\u540c\u884c\u8f66\u7684\u8981\u6c42\uff0c\u5982\u679c\u6ee1\u8db3\u5219\u53d6\u51fa\u8fd9\u4e24\u8f86\u8f66\u3002<\/li>\n<\/ol>\n<p>\uff08\u8f66\u724c\u53f71\uff0c\u8f66\u724c\u53f72\uff09\uff0c\uff08\u901a\u8fc7\u7684\u7b2c1\u4e2a\u6536\u8d39\u7ad9\uff0c\u901a\u8fc7\u7684\u7b2c5\u4e2a\u6536\u8d39\u7ad9\uff09<\/p>\n<p>\uff08\u8f66\u724c\u53f71\uff0c\u8f66\u724c\u53f73\uff09\uff0c\uff08\u901a\u8fc7\u7684\u7b2c1\u4e2a\u6536\u8d39\u7ad9\uff0c\u901a\u8fc7\u7684\u7b2c2\u4e2a\u6536\u8d39\u7ad9\uff09<\/p>\n<ol>\n<li>\u6839\u636e\u901a\u8fc7\u76f8\u540c\u6536\u8d39\u7ad9\u7684\u4e24\u8f86\u8f66\u7684\u8f66\u724c\u53f7\u805a\u5408\u6570\u636e\uff0c\u5982\u4e0b\uff1a<\/li>\n<\/ol>\n<p>\uff08\u8f66\u724c\u53f71\uff0c\u8f66\u724c\u53f72\uff09\uff0c[\uff08\u901a\u8fc7\u7684\u7b2c1\u4e2a\u6536\u8d39\u7ad9\uff0c\u901a\u8fc7\u7684\u7b2c5\u4e2a\u6536\u8d39\u7ad9\uff09\uff0c\uff08\u901a\u8fc7\u7684\u7b2c2\u4e2a\u6536\u8d39\u7ad9\uff0c\u901a\u8fc7\u7684\u7b2c6\u4e2a\u6536\u8d39\u7ad9\uff09\uff0c\uff08\u901a\u8fc7\u7684\u7b2c1\u4e2a\u6536\u8d39\u7ad9\uff0c\u901a\u8fc7\u7684\u7b2c7\u4e2a\u6536\u8d39\u7ad9\uff09\uff0c\uff08\u901a\u8fc7\u7684\u7b2c3\u4e2a\u6536\u8d39\u7ad9\uff0c\u901a\u8fc7\u7684\u7b2c8\u4e2a\u6536\u8d39\u7ad9\uff09]<\/p>\n<ol>\n<li>\u5982\u679c\u8f66\u724c\u53f71\u548c\u8f66\u724c\u53f72\u901a\u8fc7\u76f8\u540c\u6536\u8d39\u7ad9\u662f\u987a\u5e8f\u6392\u5217\u7684\uff08\u6bd4\u5982\u6536\u8d39\u7ad93\u30014\u30015\u662f\u8f66\u724c1\u901a\u8fc7\u7684\u7b2c1\u30012\u30013\u4e2a\u6536\u8d39\u7ad9\uff0c\u662f\u8f66\u724c2\u901a\u8fc7\u7684\u7b2c6\u30017\u30018\u4e2a\u6536\u8d39\u7ad9\uff09\u4e14\u6570\u91cf\u5927\u4e8e\u540c\u884c\u8f66\u8981\u6c42\u7684\u6570\u91cf\u5219\u8fd9\u4e24\u8f86\u8f66\u662f\u540c\u884c\u8f66\u3002<\/li>\n<\/ol>\n<p>\u5b9e\u73b01\u903b\u8f91\u7684\u7f3a\u70b9 \uff1a<\/p>\n<ul>\n<li>\u903b\u8f91\u590d\u6742<\/li>\n<li>\u5b9e\u73b0\u8fc7\u7a0b\u4e2dshuffle\u64cd\u4f5c\u8fc7\u591a\uff0c\u5bf9\u6027\u80fd\u5f71\u54cd\u8f83\u5927\u3002<\/li>\n<\/ul>\n<p>\u56fe12-3\u00a0\u5b9e\u73b02\u903b\u8f91<\/p>\n<p>\u00a0<\/p>\n<p>\u5b9e\u73b02\u7684\u903b\u8f91\u8bf4\u660e \uff1a<\/p>\n<ol>\n<li>\u6839\u636e\u8f66\u724c\u53f7\u805a\u5408\u8be5\u8f66\u901a\u8fc7\u7684\u6240\u6709\u6536\u8d39\u7ad9\u5e76\u6392\u5e8f\uff0c\u5904\u7406\u540e\u6570\u636e\u5982\u4e0b\uff1a<\/li>\n<\/ol>\n<p>\u8f66\u724c\u53f71\uff0c[\uff08\u901a\u8fc7\u65f6\u95f4\uff0c\u6536\u8d39\u7ad93\uff09\uff0c\uff08\u901a\u8fc7\u65f6\u95f4\uff0c\u6536\u8d39\u7ad92\uff09\uff0c\uff08\u901a\u8fc7\u65f6\u95f4\uff0c\u6536\u8d39\u7ad94\uff09\uff0c\uff08\u901a\u8fc7\u65f6\u95f4\uff0c\u6536\u8d39\u7ad95\uff09]<\/p>\n<ol>\n<li>\u6839\u636e\u540c\u884c\u8f66\u8981\u901a\u8fc7\u7684\u6536\u8d39\u7ad9\u6570\u91cf\uff08\u4f8b\u5b50\u91cc\u4e3a3\uff09\u5206\u6bb5\u8be5\u8f66\u901a\u8fc7\u7684\u6536\u8d39\u7ad9\u5e8f\u5217\uff0c\u5982\u4e0a\u9762\u7684\u6570\u636e\u88ab\u5206\u89e3\u6210 \uff1a<\/li>\n<\/ol>\n<p>\u6536\u8d39\u7ad93-&gt;\u6536\u8d39\u7ad92-&gt;\u6536\u8d39\u7ad94\uff0c \uff08\u8f66\u724c\u53f71\uff0c[\u6536\u8d39\u7ad93\u65f6\u95f4\uff0c\u6536\u8d39\u7ad92\u65f6\u95f4\uff0c\u6536\u8d39\u7ad94\u65f6\u95f4]\uff09<\/p>\n<p>\u6536\u8d39\u7ad92-&gt;\u6536\u8d39\u7ad94-&gt;\u6536\u8d39\u7ad95\uff0c \uff08\u8f66\u724c\u53f71\uff0c[\u6536\u8d39\u7ad92\u65f6\u95f4\uff0c\u6536\u8d39\u7ad94\u65f6\u95f4\uff0c\u6536\u8d39\u7ad95\u65f6\u95f4]\uff09<\/p>\n<ol>\n<li>\u628a\u901a\u8fc7\u76f8\u540c\u6536\u8d39\u7ad9\u5e8f\u5217\u7684\u8f66\u8f86\u805a\u5408\uff0c\u5982\u4e0b\uff1a<\/li>\n<\/ol>\n<p>\u6536\u8d39\u7ad93-&gt;\u6536\u8d39\u7ad92-&gt;\u6536\u8d39\u7ad94\uff0c[\uff08\u8f66\u724c\u53f71\uff0c[\u6536\u8d39\u7ad93\u65f6\u95f4\uff0c\u6536\u8d39\u7ad92\u65f6\u95f4\uff0c\u6536\u8d39\u7ad94\u65f6\u95f4]\uff09\uff0c\uff08\u8f66\u724c\u53f72\uff0c[\u6536\u8d39\u7ad93\u65f6\u95f4\uff0c\u6536\u8d39\u7ad92\u65f6\u95f4\uff0c\u6536\u8d39\u7ad94\u65f6\u95f4]\uff09\uff0c\uff08\u8f66\u724c\u53f73\uff0c[\u6536\u8d39\u7ad93\u65f6\u95f4\uff0c\u6536\u8d39\u7ad92\u65f6\u95f4\uff0c\u6536\u8d39\u7ad94\u65f6\u95f4]\uff09]<\/p>\n<ol>\n<li>\u5224\u65ad\u901a\u8fc7\u76f8\u540c\u5e8f\u5217\u6536\u8d39\u7ad9\u7684\u8f66\u8f86\u901a\u8fc7\u76f8\u540c\u6536\u8d39\u7ad9\u7684\u65f6\u95f4\u5dee\u662f\u4e0d\u662f\u6ee1\u8db3\u540c\u884c\u8f66\u7684\u8981\u6c42\uff0c\u5982\u679c\u6ee1\u8db3\u5219\u8bf4\u660e\u662f\u540c\u884c\u8f66\u3002<\/li>\n<\/ol>\n<p>\u5b9e\u73b02\u7684\u4f18\u70b9\u5982\u4e0b\uff1a<\/p>\n<ul>\n<li>\u7b80\u5316\u4e86\u5b9e\u73b0\u903b\u8f91\u3002<\/li>\n<li>\u51cf\u5c11\u4e86\u4e00\u4e2agroupByKey\uff0c\u4e5f\u5c31\u51cf\u5c11\u4e86\u4e00\u6b21shuffle\u64cd\u4f5c\uff0c\u63d0\u5347\u4e86\u6027\u80fd\u3002<\/li>\n<\/ul>\n<p>12.9.1.9\u00a0\u7ecf\u9a8c\u603b\u7ed3<\/p>\n<p>\u4f7f\u7528mapPartitions\uff0c\u6309\u6bcf\u4e2a\u5206\u533a\u8ba1\u7b97\u7ed3\u679c<\/p>\n<p>\u5982\u679c\u6bcf\u6761\u8bb0\u5f55\u7684\u5f00\u9500\u592a\u5927\uff0c\u4f8b\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2rdd.map{x=&amp;gt;conn=getDBConn;conn.write(x.toString);conn.close}<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u5219\u53ef\u4ee5\u4f7f\u7528MapPartitions\uff0c\u6309\u6bcf\u4e2a\u5206\u533a\u8ba1\u7b97\u7ed3\u679c\uff0c\u5982<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2rdd.mapPartitions(records =&amp;gt; conn.getDBConn;for(item &amp;lt;- records)<br \/>\n3write(item.toString); conn.close)<br \/>\n4<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u4f7f\u7528mapPartitions\u53ef\u4ee5\u66f4\u7075\u6d3b\u5730\u64cd\u4f5c\u6570\u636e\uff0c\u4f8b\u5982\u5bf9\u4e00\u4e2a\u5f88\u5927\u7684\u6570\u636e\u6c42TopN\uff0c\u5f53N\u4e0d\u662f\u5f88\u5927\u65f6\uff0c\u53ef\u4ee5\u5148\u4f7f\u7528mapPartitions\u5bf9\u6bcf\u4e2apartition\u6c42TopN\uff0ccollect\u7ed3\u679c\u5230\u672c\u5730\u4e4b\u540e\u518d\u505a\u6392\u5e8f\u53d6TopN\u3002\u8fd9\u6837\u76f8\u6bd4\u76f4\u63a5\u5bf9\u5168\u91cf\u6570\u636e\u505a\u6392\u5e8f\u53d6TopN\u6548\u7387\u8981\u9ad8\u5f88\u591a\u3002<\/p>\n<p>\u4f7f\u7528coalesce\u8c03\u6574\u5206\u7247\u7684\u6570\u91cf<\/p>\n<p>coalesce\u53ef\u4ee5\u8c03\u6574\u5206\u7247\u7684\u6570\u91cf\u3002coalesce\u51fd\u6570\u6709\u4e24\u4e2a\u53c2\u6570\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2coalesce(numPartitions: Int, shuffle: Boolean = false)<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u5f53shuffle\u4e3atrue\u7684\u65f6\u5019\uff0c\u51fd\u6570\u4f5c\u7528\u4e0erepartition(numPartitions: Int)\u76f8\u540c\uff0c\u4f1a\u5c06\u6570\u636e\u901a\u8fc7Shuffle\u7684\u65b9\u5f0f\u91cd\u65b0\u5206\u533a\uff1b\u5f53shuffle\u4e3afalse\u7684\u65f6\u5019\uff0c\u5219\u53ea\u662f\u7b80\u5355\u7684\u5c06\u7236RDD\u7684\u591a\u4e2apartition\u5408\u5e76\u5230\u540c\u4e00\u4e2atask\u8fdb\u884c\u8ba1\u7b97\uff0cshuffle\u4e3afalse\u65f6\uff0c\u5982\u679cnumPartitions\u5927\u4e8e\u7236RDD\u7684\u5207\u7247\u6570\uff0c\u90a3\u4e48\u5206\u533a\u4e0d\u4f1a\u91cd\u65b0\u8c03\u6574\u3002<\/p>\n<p>\u9047\u5230\u4e0b\u5217\u573a\u666f\uff0c\u53ef\u9009\u62e9\u4f7f\u7528coalesce\u7b97\u5b50\uff1a<\/p>\n<ul>\n<li>\u5f53\u4e4b\u524d\u7684\u64cd\u4f5c\u6709\u5f88\u591afilter\u65f6\uff0c\u4f7f\u7528coalesce\u51cf\u5c11\u7a7a\u8fd0\u884c\u7684\u4efb\u52a1\u6570\u91cf\u3002\u6b64\u65f6\u4f7f\u7528coalesce(numPartitions, false)\uff0cnumPartitions\u5c0f\u4e8e\u7236RDD\u5207\u7247\u6570\u3002<\/li>\n<li>\u5f53\u8f93\u5165\u5207\u7247\u4e2a\u6570\u592a\u5927\uff0c\u5bfc\u81f4\u7a0b\u5e8f\u65e0\u6cd5\u6b63\u5e38\u8fd0\u884c\u65f6\u4f7f\u7528\u3002<\/li>\n<li>\u5f53\u4efb\u52a1\u6570\u8fc7\u5927\u65f6\u5019Shuffle\u538b\u529b\u592a\u5927\u5bfc\u81f4\u7a0b\u5e8f\u6302\u4f4f\u4e0d\u52a8\uff0c\u6216\u8005\u51fa\u73b0linux\u8d44\u6e90\u53d7\u9650\u7684\u95ee\u9898\u3002\u6b64\u65f6\u9700\u8981\u5bf9\u6570\u636e\u91cd\u65b0\u8fdb\u884c\u5206\u533a\uff0c\u4f7f\u7528coalesce(numPartitions, true)\u3002<\/li>\n<\/ul>\n<p>localDir\u914d\u7f6e<\/p>\n<p>Spark\u7684Shuffle\u8fc7\u7a0b\u9700\u8981\u5199\u672c\u5730\u78c1\u76d8\uff0cShuffle\u662fSpark\u6027\u80fd\u7684\u74f6\u9888\uff0cI\/O\u662fShuffle\u7684\u74f6\u9888\u3002\u914d\u7f6e\u591a\u4e2a\u78c1\u76d8\u5219\u53ef\u4ee5\u5e76\u884c\u7684\u628a\u6570\u636e\u5199\u5165\u78c1\u76d8\u3002\u5982\u679c\u8282\u70b9\u4e2d\u6302\u8f7d\u591a\u4e2a\u78c1\u76d8\uff0c\u5219\u5728\u6bcf\u4e2a\u78c1\u76d8\u914d\u7f6e\u4e00\u4e2aSpark\u7684localDir\uff0c\u8fd9\u5c06\u6709\u6548\u5206\u6563Shuffle\u6587\u4ef6\u7684\u5b58\u653e\uff0c\u63d0\u9ad8\u78c1\u76d8I\/O\u7684\u6548\u7387\u3002\u5982\u679c\u53ea\u6709\u4e00\u4e2a\u78c1\u76d8\uff0c\u914d\u7f6e\u4e86\u591a\u4e2a\u76ee\u5f55\uff0c\u6027\u80fd\u63d0\u5347\u6548\u679c\u4e0d\u660e\u663e\u3002<\/p>\n<p>Collect\u5c0f\u6570\u636e<\/p>\n<p>\u5927\u6570\u636e\u91cf\u4e0d\u9002\u7528collect\u64cd\u4f5c\u3002<\/p>\n<p>collect\u64cd\u4f5c\u4f1a\u5c06Executor\u7684\u6570\u636e\u53d1\u9001\u5230Driver\u7aef\uff0c\u56e0\u6b64\u4f7f\u7528collect\u524d\u9700\u8981\u786e\u4fddDriver\u7aef\u5185\u5b58\u8db3\u591f\uff0c\u4ee5\u514dDriver\u8fdb\u7a0b\u53d1\u751fOutOfMemory\u5f02\u5e38\u3002\u5f53\u4e0d\u786e\u5b9a\u6570\u636e\u91cf\u5927\u5c0f\u65f6\uff0c\u53ef\u4f7f\u7528saveAsTextFile\u7b49\u64cd\u4f5c\u628a\u6570\u636e\u5199\u5165HDFS\u4e2d\u3002\u53ea\u6709\u5728\u80fd\u591f\u5927\u81f4\u786e\u5b9a\u6570\u636e\u5927\u5c0f\u4e14driver\u5185\u5b58\u5145\u8db3\u7684\u65f6\u5019\uff0c\u624d\u80fd\u4f7f\u7528collect\u3002<\/p>\n<p>\u4f7f\u7528reduceByKey<\/p>\n<p>reduceByKey\u4f1a\u5728Map\u7aef\u505a\u672c\u5730\u805a\u5408\uff0c\u4f7f\u5f97Shuffle\u8fc7\u7a0b\u66f4\u52a0\u5e73\u7f13\uff0c\u800cgroupByKey\u7b49Shuffle\u64cd\u4f5c\u4e0d\u4f1a\u5728Map\u7aef\u505a\u805a\u5408\u3002\u56e0\u6b64\u80fd\u4f7f\u7528reduceByKey\u7684\u5730\u65b9\u5c3d\u91cf\u4f7f\u7528\u8be5\u7b97\u5b50\uff0c\u907f\u514d\u51fa\u73b0groupByKey().map(x=&gt;(x._1,x._2.size))\u8fd9\u7c7b\u5b9e\u73b0\u65b9\u5f0f\u3002<\/p>\n<p>\u5e7f\u64admap\u4ee3\u66ff\u6570\u7ec4<\/p>\n<p>\u5f53\u6bcf\u6761\u8bb0\u5f55\u9700\u8981\u67e5\u8868\uff0c\u5982\u679c\u662fDriver\u7aef\u7528\u5e7f\u64ad\u65b9\u5f0f\u4f20\u9012\u7684\u6570\u636e\uff0c\u6570\u636e\u7ed3\u6784\u4f18\u5148\u91c7\u7528set\/map\u800c\u4e0d\u662fIterator\uff0c\u56e0\u4e3aSet\/Map\u7684\u67e5\u8be2\u901f\u7387\u63a5\u8fd1O(1)\uff0c\u800cIterator\u662fO(n)\u3002<\/p>\n<p>\u6570\u636e\u503e\u659c<\/p>\n<p>\u5f53\u6570\u636e\u53d1\u751f\u503e\u659c\uff08\u67d0\u4e00\u90e8\u5206\u6570\u636e\u91cf\u7279\u522b\u5927\uff09\uff0c\u867d\u7136\u6ca1\u6709GC\uff08Gabage Collection\uff0c\u5783\u573e\u56de\u6536\uff09\uff0c\u4f46\u662ftask\u6267\u884c\u65f6\u95f4\u4e25\u91cd\u4e0d\u4e00\u81f4\u3002<\/p>\n<ul>\n<li>\u9700\u8981\u91cd\u65b0\u8bbe\u8ba1key\uff0c\u4ee5\u66f4\u5c0f\u7c92\u5ea6\u7684key\u4f7f\u5f97task\u5927\u5c0f\u5408\u7406\u5316\u3002<\/li>\n<li>\u4fee\u6539\u5e76\u884c\u5ea6\u3002<\/li>\n<\/ul>\n<p>\u4f18\u5316\u6570\u636e\u7ed3\u6784<\/p>\n<ul>\n<li>\u628a\u6570\u636e\u6309\u5217\u5b58\u653e\uff0c\u8bfb\u53d6\u6570\u636e\u65f6\u5c31\u53ef\u4ee5\u53ea\u626b\u63cf\u9700\u8981\u7684\u5217\u3002<\/li>\n<li>\u4f7f\u7528Hash Shuffle\u65f6\uff0c\u901a\u8fc7\u8bbe\u7f6espark.shuffle.consolidateFiles\u4e3atrue\uff0c\u6765\u5408\u5e76shuffle\u4e2d\u95f4\u6587\u4ef6\uff0c\u51cf\u5c11shuffle\u6587\u4ef6\u7684\u6570\u91cf\uff0c\u51cf\u5c11\u6587\u4ef6IO\u64cd\u4f5c\u4ee5\u63d0\u5347\u6027\u80fd\u3002\u6700\u7ec8\u6587\u4ef6\u6570\u4e3areduce tasks\u6570\u76ee\u3002<\/li>\n<\/ul>\n<p>12.9.1.10\u00a0\u4f18\u5316\u8fd4\u56de\u67e5\u8be2\u7ed3\u679c\u8fd4\u56de\u5927\u91cf\u6570\u636e\u7684\u573a\u666f<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u4f7f\u7528\u547d\u4ee4\u63d0\u4ea4\u4efb\u52a1\u65f6\uff0c\u5982\u679c\u4efb\u52a1\u4e2d\u5305\u542b\u4e86\u67e5\u8be2\u7ed3\u679c\u8fd4\u56de\u5927\u91cf\u6570\u636e\u7684\u64cd\u4f5c\uff0c\u7531\u4e8e\u6b64\u7c7b\u64cd\u4f5c\u4f1a\u5728driver\u7aef\u5360\u7528\u5927\u91cf\u5185\u5b58\u8fdb\u884c\u6570\u636e\u5b58\u653e\uff0c\u5219\u5b58\u5728\u5bfc\u81f4\u5185\u5b58\u6ea2\u51fa\u7684\u98ce\u9669\uff0c\u56e0\u6b64\u9700\u8981\u8fdb\u884c\u4f18\u5316\u6765\u652f\u6301\u8be5\u573a\u666f\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u5728\u5ba2\u6237\u7aef\u7684\u201cspark-defaults.conf\u201d\u914d\u7f6e\u6587\u4ef6\u4e2d\u8c03\u6574\u5982\u4e0b\u53c2\u6570\u3002<\/p>\n<p>\u886812-25\u00a0\u53c2\u6570\u8bf4\u660e<\/p>\n<p>spark.sql.bigdata.thriftServer.useHdfsCollect<br \/>\ndriver\u7aef\u8bfb\u53d6\u6570\u636e\u65f6\u7684\u65b9\u5f0f\uff0c\u5f53\u914d\u7f6e\u4e3afalse\u65f6\uff0c\u8bfb\u53d6\u7684\u6570\u636e\u4f1a\u5b58\u653e\u5728\u5185\u5b58\u4e2d\uff0c\u9002\u5408\u6570\u636e\u91cf\u4e0d\u5927\uff08\u6570\u636e\u91cf\u5c0f\u4e8eDriver\u7aef\u8bbe\u7f6e\u7684\u5185\u5b58\u5927\u5c0f\uff09\u7684\u60c5\u51b5\u3002 \u5982\u679c\u8fd4\u56de\u7684\u6570\u636e\u91cf\u5927\uff08\u6570\u636e\u91cf\u5927\u4e8eDriver\u7aef\u8bbe\u7f6e\u7684\u5185\u5b58\u5927\u5c0f\uff09\uff0c\u9700\u8981\u628a\u6b64\u503c\u914d\u7f6e\u6210true\uff0c\u7531\u4e8e\u5148\u4fdd\u5b58\u6210HDFS\u6587\u4ef6\uff0c\u518d\u5206\u6279\u8bfb\u53d6\uff0c\u6027\u80fd\u6bd4\u914d\u7f6e\u6210false\u6162\u3002<br \/>\nfalse<\/p>\n<h3>12.9.2\u00a0SQL\u548cDataFrame\u8c03\u4f18<\/h3>\n<p>12.9.2.1\u00a0Spark SQL join\u4f18\u5316<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>Spark SQL\u4e2d\uff0c\u5f53\u5bf9\u4e24\u4e2a\u8868\u8fdb\u884cjoin\u64cd\u4f5c\u65f6\uff0c\u5229\u7528Broadcast\u7279\u6027\uff08\u89c1\u201c\u4f7f\u7528\u5e7f\u64ad\u53d8\u91cf\u201d\u7ae0\u8282\uff09\uff0c\u5c06\u5c0f\u8868BroadCast\u5230\u5404\u4e2a\u8282\u70b9\u4e0a\uff0c\u4ece\u800c\u8f6c\u53d8\u6210\u975eshuffle\u64cd\u4f5c\uff0c\u63d0\u9ad8\u4efb\u52a1\u6267\u884c\u6027\u80fd\u3002<\/p>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u8fd9\u91ccjoin\u64cd\u4f5c\uff0c\u53ea\u6307inner join\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u5728Spark SQL\u4e2d\u8fdb\u884cJoin\u64cd\u4f5c\u65f6\uff0c\u53ef\u4ee5\u6309\u7167\u4ee5\u4e0b\u6b65\u9aa4\u8fdb\u884c\u4f18\u5316\u3002\u4e3a\u4e86\u65b9\u4fbf\u8bf4\u660e\uff0c\u8bbe\u8868A\u548c\u8868B\uff0c\u4e14A\u3001B\u8868\u90fd\u6709\u4e2a\u540d\u4e3aname\u7684\u5217\u3002\u5bf9A\u3001B\u8868\u8fdb\u884cjoin\u64cd\u4f5c\u3002<\/p>\n<ol>\n<li>\u4f30\u8ba1\u8868\u7684\u5927\u5c0f\u3002<\/li>\n<\/ol>\n<p>\u6839\u636e\u6bcf\u6b21\u52a0\u8f7d\u6570\u636e\u7684\u5927\u5c0f\uff0c\u6765\u4f30\u8ba1\u8868\u5927\u5c0f\u3002<\/p>\n<p>\u4e5f\u53ef\u4ee5\u5728Hive\u7684\u6570\u636e\u5e93\u5b58\u50a8\u8def\u5f84\u4e0b\u76f4\u63a5\u67e5\u770b\u8868\u7684\u5927\u5c0f\u3002\u9996\u5148\u5728Spark\u7684\u914d\u7f6e\u6587\u4ef6hive-site.xml\u4e2d\uff0c\u67e5\u770bHive\u7684\u6570\u636e\u5e93\u8def\u5f84\u7684\u914d\u7f6e\uff0c\u9ed8\u8ba4\u4e3a\u201c\/user\/hive\/warehouse\u201d\u3002Spark\u670d\u52a1\u591a\u5b9e\u4f8b\u9ed8\u8ba4\u6570\u636e\u5e93\u8def\u5f84\u4e3a\u201c\/user\/<em>hive<\/em>\/warehouse\u201d\uff0c\u4f8b\u5982\u201c\/user\/hive1\/warehouse\u201d\u3002<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/>6<br \/>7<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2&amp;lt;property&amp;gt;<br \/>\n3 &nbsp; &amp;lt;name&amp;gt;hive.metastore.warehouse.dir&amp;lt;\/name&amp;gt;<br \/>\n4 &nbsp;&amp;lt;value&amp;gt;${test.warehouse.dir}&amp;lt;\/value&amp;gt;<br \/>\n5 &nbsp;&amp;lt;description&amp;gt;&amp;lt;\/description&amp;gt;<br \/>\n6&amp;lt;\/property&amp;gt;<br \/>\n7<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u7136\u540e\u901a\u8fc7hadoop\u547d\u4ee4\u67e5\u770b\u5bf9\u5e94\u8868\u7684\u5927\u5c0f\u3002\u5982\u67e5\u770b\u8868A\u7684\u5927\u5c0f\u547d\u4ee4\u4e3a\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2hadoop fs -du -s -h ${test.warehouse.dir}\/a<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/><\/div><\/td><td><div class=\"text codecolorer\">1 \u00a0\u8bf4\u660e\uff1a <br \/>\n2<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u8fdb\u884c\u5e7f\u64ad\u64cd\u4f5c\uff0c\u9700\u8981\u81f3\u5c11\u6709\u4e00\u4e2a\u8868\u4e0d\u662f\u7a7a\u8868\u3002<\/p>\n<ol>\n<li>\u914d\u7f6e\u81ea\u52a8\u5e7f\u64ad\u7684\u9608\u503c\u3002<\/li>\n<\/ol>\n<p>Spark\u4e2d\uff0c\u5224\u65ad\u8868\u662f\u5426\u5e7f\u64ad\u7684\u9608\u503c\u4e3a67108864\uff08\u537364M\uff09\u3002\u5982\u679c\u4e24\u4e2a\u8868\u7684\u5927\u5c0f\u81f3\u5c11\u6709\u4e00\u4e2a\u5c0f\u4e8e64M\u65f6\uff0c\u53ef\u4ee5\u8df3\u8fc7\u8be5\u6b65\u9aa4\u3002<\/p>\n<p>\u81ea\u52a8\u5e7f\u64ad\u9608\u503c\u7684\u914d\u7f6e\u53c2\u6570\u4ecb\u7ecd\uff0c\u89c1\u886812-26\u3002<br \/>\n\u886812-26\u00a0\u53c2\u6570\u4ecb\u7ecd<\/p>\n<p>spark.sql.autoBroadcastJoinThreshold<br \/>\n67108864<br \/>\n\u5f53\u8fdb\u884cjoin\u64cd\u4f5c\u65f6\uff0c\u914d\u7f6e\u5e7f\u64ad\u7684\u6700\u5927\u503c\uff1b\u5f53\u8868\u7684\u5b57\u8282\u6570\u5c0f\u4e8e\u8be5\u503c\u65f6\u4fbf\u8fdb\u884c\u5e7f\u64ad\u3002\u5f53\u914d\u7f6e\u4e3a-1\u65f6\uff0c\u5c06\u4e0d\u8fdb\u884c\u5e7f\u64ad\u3002 \u53c2\u89c1<a href=\"https:\/\/spark.apache.org\/docs\/latest\/sql-programming-guide.html\">https:\/\/spark.apache.org\/docs\/latest\/sql-programming-guide.html<\/a><\/p>\n<p>\u914d\u7f6e\u81ea\u52a8\u5e7f\u64ad\u9608\u503c\u7684\u65b9\u6cd5\uff1a<\/p>\n<ul>\n<li>\u5728Spark\u7684\u914d\u7f6e\u6587\u4ef6\u201cspark-defaults.conf\u201d\u4e2d\uff0c\u8bbe\u7f6e\u201cspark.sql.autoBroadcastJoinThreshold\u201d\u7684\u503c\u3002\u5176\u4e2d\uff0c<!-- -->&lt;size&gt;<!-- -->\u6839\u636e\u573a\u666f\u800c\u5b9a\uff0c\u4f46\u8981\u6c42\u8be5\u503c\u81f3\u5c11\u6bd4\u5176\u4e2d\u4e00\u4e2a\u8868\u5927\u3002<\/li>\n<\/ul>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2spark.sql.autoBroadcastJoinThreshold = &amp;lt;size&amp;gt;<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<ul>\n<li>\u5229\u7528Hive CLI\u547d\u4ee4\uff0c\u8bbe\u7f6e\u9608\u503c\u3002\u5728\u8fd0\u884cJoin\u64cd\u4f5c\u65f6\uff0c\u63d0\u524d\u8fd0\u884c\u4e0b\u9762\u8bed\u53e5<\/li>\n<\/ul>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2SET spark.sql.autoBroadcastJoinThreshold=&amp;lt;size&amp;gt;<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u5176\u4e2d\uff0c<!-- -->&lt;size&gt;<!-- -->\u6839\u636e\u573a\u666f\u800c\u5b9a\uff0c\u4f46\u8981\u6c42\u8be5\u503c\u81f3\u5c11\u6bd4\u5176\u4e2d\u4e00\u4e2a\u8868\u5927\u3002<\/p>\n<ul>\n<li>\n<p>\uff08\u53ef\u9009\uff09\u5982\u4e0b\u4e24\u79cd\u573a\u666f\uff0c\u9700\u8981\u6267\u884cAnalyze\u547d\u4ee4\uff08<em><strong>ANALYZE TABLE tableName COMPUTE STATISTICS noscan;<\/strong><\/em>\uff09\u66f4\u65b0\u8868\u5143\u6570\u636e\u540e\u8fdb\u884c\u5e7f\u64ad\u3002<\/p>\n<\/li>\n<li>\n<p>\u9700\u8981\u5e7f\u64ad\u7684\u8868\u662f\u5206\u533a\u8868\uff0c\u65b0\u5efa\u8868\u4e14\u6587\u4ef6\u7c7b\u578b\u4e3a\u975eParquet\u6587\u4ef6\u7c7b\u578b\u3002<\/p>\n<ul>\n<li>\u9700\u8981\u5e7f\u64ad\u7684\u8868\u662f\u5206\u533a\u8868\uff0c\u66f4\u65b0\u8868\u6570\u636e\u540e\u3002<\/li>\n<\/ul>\n<\/li>\n<li>\n<p>\u8fdb\u884cjoin\u64cd\u4f5c\u3002<\/p>\n<\/li>\n<\/ul>\n<p>\u8fd9\u65f6join\u7684\u4e24\u4e2atable\uff0c\u81f3\u5c11\u6709\u4e2a\u8868\u662f\u5c0f\u4e8e\u9608\u503c\u7684\u3002<\/p>\n<p>\u5982\u679cA\u8868\u548cB\u8868\u90fd\u5c0f\u4e8e\u9608\u503c\uff0c\u4e14A\u8868\u7684\u5b57\u8282\u6570\u5c0f\u4e8eB\u8868\u65f6\uff0c\u5219\u8fd0\u884cB join A\uff0c\u5982<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2SELECT A.name FROM B JOIN A ON A.name = B.name\uff1b<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u5426\u5219\u8fd0\u884cA join B\u3002<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2SELECT A.name FROM A JOIN B ON A.name = B.name\uff1b<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<ul>\n<li>\u4f7f\u7528Executor\u5e7f\u64ad\u51cf\u5c11Driver\u5185\u5b58\u538b\u529b\u3002<\/li>\n<\/ul>\n<p>\u9ed8\u8ba4\u7684BroadCastJoin\u4f1a\u5c06\u5c0f\u8868\u7684\u5185\u5bb9\uff0c\u5168\u90e8\u6536\u96c6\u5230Driver\u4e2d\uff0c\u56e0\u6b64\u9700\u8981\u9002\u5f53\u7684\u8c03\u5927Driver\u7684\u5185\u5b58\u3002\u5185\u5b58\u589e\u52a0\u7684\u8ba1\u7b97\u516c\u5f0f\u4e3a\uff1a\u201cspark.sql.autoBroadcastJoinThreshold * the number of broadcast table * 2\u201d\u3002\u5f53\u5e7f\u64ad\u4efb\u52a1\u6bd4\u8f83\u9891\u7e41\u7684\u65f6\u5019\uff0cDriver\u6709\u53ef\u80fd\u56e0\u4e3aOOM\u800c\u5f02\u5e38\u9000\u51fa\u3002<\/p>\n<p>\u6b64\u65f6\uff0c\u53ef\u4ee5\u5f00\u542fExecutor\u5e7f\u64ad\uff0c\u5728\u5ba2\u6237\u7aef\u201cspark-defaults.conf\u201d\u6587\u4ef6\u4e2d\u914d\u7f6eExecutor\u5e7f\u64ad\u53c2\u6570\u201cspark.sql.bigdata.useExecutorBroadcast\u201d\u4e3a\u201ctrue\u201d\uff0c\u51cf\u5c11Driver\u5185\u5b58\u538b\u529b\u3002<br \/>\n\u886812-27\u00a0\u53c2\u6570\u4ecb\u7ecd<\/p>\n<p>spark.sql.bigdata.useExecutorBroadcast<br \/>\n\u8bbe\u7f6e\u4e3atrue\u65f6\uff0c\u4f7f\u7528Executor\u5e7f\u64ad\uff0c\u5c06\u8868\u6570\u636e\u7f13\u5b58\u5728Executor\u4e2d\uff0c\u800c\u4e0d\u662f\u653e\u5728Driver\u4e4b\u4e2d\uff0c\u51cf\u5c11Spark Driver\u5185\u5b58\u7684\u538b\u529b\u3002<br \/>\ntrue<\/p>\n<p>\u53c2\u8003\u4fe1\u606f<\/p>\n<p>\u5c0f\u8868\u6267\u884c\u8d85\u65f6\uff0c\u5bfc\u81f4\u4efb\u52a1\u7ed3\u675f\u3002<\/p>\n<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0cBroadCastJoin\u53ea\u5141\u8bb8\u5c0f\u8868\u8ba1\u7b975\u5206\u949f\uff0c\u8d85\u8fc75\u5206\u949f\u8be5\u4efb\u52a1\u4f1a\u51fa\u73b0\u8d85\u65f6\u5f02\u5e38\uff0c\u800c\u8fd9\u4e2a\u65f6\u5019\u5c0f\u8868\u7684broadcast\u4efb\u52a1\u4f9d\u7136\u5728\u6267\u884c\uff0c\u9020\u6210\u8d44\u6e90\u6d6a\u8d39\u3002<\/p>\n<p>\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0c\u6709\u4e24\u79cd\u65b9\u5f0f\u5904\u7406\uff1a<\/p>\n<ul>\n<li>\u8c03\u6574\u201cspark.sql.broadcastTimeout\u201d\u7684\u6570\u503c\uff0c\u52a0\u5927\u8d85\u65f6\u7684\u65f6\u95f4\u9650\u5236\u3002<\/li>\n<li>\u964d\u4f4e\u201cspark.sql.autoBroadcastJoinThreshold\u201d\u7684\u6570\u503c\uff0c\u4e0d\u4f7f\u7528BroadCastJoin\u7684\u4f18\u5316\u3002<\/li>\n<\/ul>\n<p>12.9.2.2\u00a0\u4f18\u5316\u6570\u636e\u503e\u659c\u573a\u666f\u4e0b\u7684Spark SQL\u6027\u80fd<\/p>\n<p>\u914d\u7f6e\u573a\u666f<\/p>\n<p>\u5728Spark SQL\u591a\u8868Join\u7684\u573a\u666f\u4e0b\uff0c\u4f1a\u5b58\u5728\u5173\u8054\u952e\u4e25\u91cd\u503e\u659c\u7684\u60c5\u51b5\uff0c\u5bfc\u81f4Hash\u5206\u6876\u540e\uff0c\u90e8\u5206\u6876\u4e2d\u7684\u6570\u636e\u8fdc\u9ad8\u4e8e\u5176\u5b83\u5206\u6876\u3002\u6700\u7ec8\u5bfc\u81f4\u90e8\u5206Task\u8fc7\u91cd\uff0c\u8dd1\u5f97\u5f88\u6162\uff1b\u5176\u5b83Task\u8fc7\u8f7b\uff0c\u8dd1\u5f97\u5f88\u5feb\u3002\u4e00\u65b9\u9762\uff0c\u6570\u636e\u91cf\u5927Task\u8fd0\u884c\u6162\uff0c\u4f7f\u5f97\u8ba1\u7b97\u6027\u80fd\u4f4e\uff1b\u53e6\u4e00\u65b9\u9762\uff0c\u6570\u636e\u91cf\u5c11\u7684Task\u5728\u8fd0\u884c\u5b8c\u6210\u540e\uff0c\u5bfc\u81f4\u5f88\u591aCPU\u7a7a\u95f2\uff0c\u9020\u6210CPU\u8d44\u6e90\u6d6a\u8d39\u3002<\/p>\n<p>\u901a\u8fc7\u5982\u4e0b\u914d\u7f6e\u9879\u53ef\u5c06\u90e8\u5206\u6570\u636e\u91c7\u7528Broadcast\u65b9\u5f0f\u5206\u53d1\uff0c\u4ee5\u4fbf\u5747\u8861Task\uff0c\u63d0\u9ad8CPU\u8d44\u6e90\u7684\u5229\u7528\u7387\uff0c\u4ece\u800c\u63d0\u9ad8\u6027\u80fd\u3002<\/p>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u672a\u4ea7\u751f\u503e\u659c\u7684\u6570\u636e\uff0c\u5c06\u91c7\u7528\u539f\u6709\u65b9\u5f0f\u8fdb\u884c\u5206\u6876\u5e76\u8fd0\u884c\u3002<\/p>\n<p>\u4f7f\u7528\u7ea6\u675f\uff1a<\/p>\n<ol>\n<li>\u4e24\u8868\u4e2d\u503e\u659c\u7684\u5173\u8054\u952e\u4e0d\u540c\u3002\uff08\u4f8b\u5982\uff1aA join B on A.name = B.name\uff0c\u5982\u679cA\u4e2dname = &quot;zhangsan&quot;\u503e\u659c\uff0c\u90a3\u4e48B\u4e2dname = &quot;zhangsan&quot;\u4e0d\u80fd\u503e\u659c\uff0c\u5176\u4ed6\u7684\u5982name = &quot;lisi&quot;\u53ef\u4ee5\u503e\u659c\u3002\uff09<\/li>\n<li>\u53ea\u652f\u6301\u4e24\u8868\u95f4\u7684Join\u3002<\/li>\n<li>TS\uff08TableScan\uff09\u5230Join\u4e4b\u95f4\u7684\u64cd\u4f5c\u53ea\u652f\u6301TS\uff08TableScan\uff09\u3001FIL\uff08Filter\uff09\u3001SEL\uff08Select\uff09\u3002<\/li>\n<\/ol>\n<p>\u914d\u7f6e\u63cf\u8ff0<\/p>\n<p>\u5728\u5ba2\u6237\u7aef\u7684\u201cspark-defaults.conf\u201d\u914d\u7f6e\u6587\u4ef6\u4e2d\u8c03\u6574\u5982\u4e0b\u53c2\u6570\u3002<\/p>\n<p>\u886812-28\u00a0\u53c2\u6570\u8bf4\u660e<\/p>\n<p>spark.sql.planner.skewJoin<br \/>\n\u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u6570\u636e\u503e\u659c\u4f18\u5316\u3002\u201ctrue\u201d\u8868\u793a\u5f00\u542f\u3002\u5f00\u542f\u540e\uff0c\u4f1a\u57fa\u4e8e\u201cspark.sql.planner.skewJoin.threshold\u201d\u53c2\u6570\u8bc6\u522b\u51fa\u503e\u659c\u5173\u952e\uff0c\u7cfb\u7edf\u4f1a\u5c06\u8fd9\u90e8\u5206\u6570\u636e\u91c7\u7528Broadcast\u65b9\u5f0f\uff0c\u53ef\u4ee5\u907f\u514d\u6570\u636e\u503e\u659c\uff0c\u63d0\u5347CPU\u5229\u7528\u7387\uff0c\u4ece\u800c\u63d0\u5347\u6027\u80fd\u3002<br \/>\nfalse<br \/>\nspark.sql.planner.skewJoin.threshold<br \/>\n\u7528\u4e8e\u5224\u65ad\u662f\u5426\u5b58\u5728\u6570\u636e\u503e\u659c\u7684\u9608\u503c\u3002\u5f53\u5b58\u5728\u5173\u8054\u952e\u7684\u4e2a\u6570\u5927\u4e8e\u8be5\u9608\u503c\uff0c\u5219\u5b58\u5728\u6570\u636e\u503e\u659c\uff0c\u8be5\u5173\u8054\u952e\u4e3a\u503e\u659c\u5173\u8054\u952e\u3002<br \/>\n100000<\/p>\n<p>12.9.2.3\u00a0\u4f18\u5316\u5c0f\u6587\u4ef6\u573a\u666f\u4e0b\u7684Spark SQL\u6027\u80fd<\/p>\n<p>\u914d\u7f6e\u573a\u666f<\/p>\n<p>Spark SQL\u7684\u8868\u4e2d\uff0c\u7ecf\u5e38\u4f1a\u5b58\u5728\u5f88\u591a\u5c0f\u6587\u4ef6\uff08\u5927\u5c0f\u8fdc\u5c0f\u4e8eHDFS\u5757\u5927\u5c0f\uff09\uff0c\u6bcf\u4e2a\u5c0f\u6587\u4ef6\u9ed8\u8ba4\u5bf9\u5e94Spark\u4e2d\u7684\u4e00\u4e2aPartition\uff0c\u4e5f\u5c31\u662f\u4e00\u4e2aTask\u3002\u5728\u5f88\u591a\u5c0f\u6587\u4ef6\u573a\u666f\u4e0b\uff0cSpark\u4f1a\u8d77\u5f88\u591aTask\u3002\u5f53SQL\u903b\u8f91\u4e2d\u5b58\u5728Shuffle\u64cd\u4f5c\u65f6\uff0c\u4f1a\u5927\u5927\u589e\u52a0hash\u5206\u6876\u6570\uff0c\u4e25\u91cd\u5f71\u54cd\u6027\u80fd\u3002<\/p>\n<p>\u5728\u5c0f\u6587\u4ef6\u573a\u666f\u4e0b\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u914d\u7f6e\u624b\u52a8\u6307\u5b9a\u6bcf\u4e2aTask\u7684\u6570\u636e\u91cf\uff08Split Size\uff09\uff0c\u786e\u4fdd\u4e0d\u4f1a\u4ea7\u751f\u8fc7\u591a\u7684Task\uff0c\u63d0\u9ad8\u6027\u80fd\u3002<\/p>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u5f53SQL\u903b\u8f91\u4e2d\u4e0d\u5305\u542bShuffle\u64cd\u4f5c\u65f6\uff0c\u8bbe\u7f6e\u6b64\u914d\u7f6e\u9879\uff0c\u4e0d\u4f1a\u6709\u660e\u663e\u7684\u6027\u80fd\u63d0\u5347\u3002<\/p>\n<p>\u914d\u7f6e\u63cf\u8ff0<\/p>\n<p>\u5728\u5ba2\u6237\u7aef\u7684\u201cspark-defaults.conf\u201d\u914d\u7f6e\u6587\u4ef6\u4e2d\u8c03\u6574\u5982\u4e0b\u53c2\u6570\u3002<\/p>\n<p>\u886812-29\u00a0\u53c2\u6570\u8bf4\u660e<\/p>\n<p>spark.sql.small.file.combine<br \/>\n\u7528\u4e8e\u8bbe\u7f6e\u662f\u5426\u5f00\u542f\u5c0f\u6587\u4ef6\u4f18\u5316\u3002\u201ctrue\u201d\u8868\u793a\u5f00\u542f\u3002\u5f00\u542f\u540e\uff0c\u53ef\u4ee5\u907f\u514d\u8fc7\u591a\u7684\u5c0fTask\u3002<br \/>\nfalse<br \/>\nspark.sql.small.file.split.size<br \/>\n\u5408\u5e76\u5c0f\u6587\u4ef6\u540e\uff0c\u7528\u4e8e\u6307\u5b9a\u5355\u4e2aTask\u671f\u671b\u7684\u6570\u636e\u91cf\u3002 \u5355\u4f4d\uff1aByte<br \/>\n256000000<\/p>\n<p>12.9.2.4\u00a0INSERT&#8230;SELECT\u64cd\u4f5c\u8c03\u4f18<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5728\u4ee5\u4e0b\u51e0\u79cd\u60c5\u51b5\u4e0b\uff0c\u6267\u884cINSERT&#8230;SELECT\u64cd\u4f5c\u53ef\u4ee5\u8fdb\u884c\u4e00\u5b9a\u7684\u8c03\u4f18\u64cd\u4f5c\u3002<\/p>\n<ul>\n<li>\u67e5\u8be2\u7684\u6570\u636e\u662f\u5927\u91cf\u7684\u5c0f\u6587\u4ef6\u3002<\/li>\n<li>\u67e5\u8be2\u7684\u6570\u636e\u662f\u8f83\u591a\u7684\u5927\u6587\u4ef6\u3002<\/li>\n<li>\u5728beeline\/JDBCServer\u6a21\u5f0f\u4e0b\u4f7f\u7528\u975espark\u7528\u6237\u64cd\u4f5c\u3002<\/li>\n<\/ul>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u53ef\u5bf9INSERT&#8230;SELECT\u64cd\u4f5c\u505a\u5982\u4e0b\u7684\u8c03\u4f18\u64cd\u4f5c\u3002<\/p>\n<ul>\n<li>\u5982\u679c\u5efa\u7684\u662fHive\u8868\uff0c\u5c06\u5b58\u50a8\u7c7b\u578b\u8bbe\u4e3aParquet\uff0c\u4ece\u800c\u51cf\u5c11\u6267\u884cINSERT&#8230;SELECT\u8bed\u53e5\u7684\u65f6\u95f4\u3002<\/li>\n<li>\u5efa\u8bae\u4f7f\u7528spark-sql\u6216\u8005\u5728beeline\/JDBCServer\u6a21\u5f0f\u4e0b\u4f7f\u7528spark\u7528\u6237\u6765\u6267\u884cINSERT&#8230;SELECT\u64cd\u4f5c\uff0c\u907f\u514d\u6267\u884c\u66f4\u6539\u6587\u4ef6owner\u7684\u64cd\u4f5c\uff0c\u4ece\u800c\u51cf\u5c11\u6267\u884cINSERT&#8230;SELECT\u8bed\u53e5\u7684\u65f6\u95f4\u3002\u00a0\u8bf4\u660e\uff1a<\/li>\n<\/ul>\n<p>\u5728beeline\/JDBCServer\u6a21\u5f0f\u4e0b\uff0cexecutor\u7684\u7528\u6237\u8ddfdriver\u662f\u4e00\u81f4\u7684\uff0cdriver\u662fJDBCServer\u670d\u52a1\u7684\u4e00\u90e8\u5206\uff0c\u662f\u7531spark\u7528\u6237\u542f\u52a8\u7684\uff0c\u56e0\u6b64\u5176\u7528\u6237\u4e5f\u662fspark\u7528\u6237\uff0c\u4e14\u5f53\u524d\u65e0\u6cd5\u5b9e\u73b0\u5728\u8fd0\u884c\u65f6\u5c06beeline\u7aef\u7684\u7528\u6237\u900f\u4f20\u5230executor\uff0c\u56e0\u6b64\u4f7f\u7528\u975espark\u7528\u6237\u65f6\u9700\u8981\u5bf9\u6587\u4ef6\u8fdb\u884c\u66f4\u6539owner\u4e3abeeline\u7aef\u7684\u7528\u6237\uff0c\u5373\u5b9e\u9645\u7528\u6237\u3002<\/p>\n<ul>\n<li>\u5982\u679c\u67e5\u8be2\u7684\u6570\u636e\u662f\u5927\u91cf\u7684\u5c0f\u6587\u4ef6\u5c06\u4f1a\u4ea7\u751f\u5927\u91cfmap\u64cd\u4f5c\uff0c\u4ece\u800c\u5bfc\u81f4\u8f93\u51fa\u5b58\u5728\u5927\u91cf\u7684\u5c0f\u6587\u4ef6\uff0c\u5728\u6267\u884c\u91cd\u547d\u540d\u6587\u4ef6\u64cd\u4f5c\u65f6\u5c06\u4f1a\u8017\u8d39\u8f83\u591a\u65f6\u95f4\uff0c\u6b64\u65f6\u53ef\u4ee5\u901a\u8fc7\u8bbe\u7f6espark.sql.small.file.combine = true\u6765\u5f00\u542f\u5c0f\u6587\u4ef6\u5408\u5e76\u529f\u80fd\u6765\u51cf\u5c11\u8f93\u51fa\u6587\u4ef6\u6570\uff0c\u51cf\u5c11\u6267\u884c\u91cd\u547d\u540d\u6587\u4ef6\u64cd\u4f5c\u7684\u65f6\u95f4\uff0c\u4ece\u800c\u51cf\u5c11\u6267\u884cINSERT&#8230;SELECT\u8bed\u53e5\u7684\u65f6\u95f4\u3002<\/li>\n<li>\u5982\u679c\u67e5\u8be2\u7684\u6570\u636e\u662f\u8f83\u591a\u7684\u5927\u6587\u4ef6\uff0c\u90a3\u4e48\u6267\u884c\u8be5\u64cd\u4f5c\u7684\u8f93\u51fa\u6587\u4ef6\u4e5f\u4f1a\u5f88\u591a\uff0c\u5f53\u8fd9\u4e2a\u91cf\u7ea7\u8fbe\u5230\u6570\u5341\u4e07\u65f6\uff0c\u6b64\u65f6\u53ef\u4ee5\u901a\u8fc7\u8bbe\u7f6espark.sql.small.file.combine = true\u6765\u5f00\u542f\u5c0f\u6587\u4ef6\u5408\u5e76\u529f\u80fd\uff0c\u540c\u65f6\u8bbe\u7f6espark.sql.small.file.split.size\u4e3a\u4e00\u4e2a\u8f83\u5408\u7406\u7684\u503c\uff0c\u63a7\u5236\u8f93\u51fa\u6587\u4ef6\u5927\u5c0f\uff0c\u51cf\u5c0f\u8f93\u51fa\u6587\u4ef6\u4e2a\u6570\uff08\u539f\u5219\u662f\u786e\u4fddreduce\u4efb\u52a1\u80fd\u591f\u5145\u5206\u5229\u7528\u96c6\u7fa4\u8d44\u6e90\uff0c\u5426\u5219\u4f1a\u589e\u52a0\u5199\u6587\u4ef6\u7684\u65f6\u95f4\uff09\uff0c\u51cf\u5c11\u6267\u884c\u91cd\u547d\u540d\u6587\u4ef6\u7684\u65f6\u95f4\uff0c\u4ece\u800c\u51cf\u5c11\u6267\u884cINSERT&#8230;SELECT\u8bed\u53e5\u7684\u65f6\u95f4\u3002<\/li>\n<\/ul>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u4e0a\u8ff0\u4f18\u5316\u64cd\u4f5c\u5e76\u4e0d\u80fd\u89e3\u51b3\u5168\u90e8\u7684\u6027\u80fd\u95ee\u9898\uff0c\u5bf9\u4e8e\u4ee5\u4e0b\u4e24\u79cd\u573a\u666f\u4ecd\u7136\u9700\u8981\u8f83\u591a\u65f6\u95f4\uff1a<\/p>\n<ul>\n<li>\u5bf9\u4e8e\u52a8\u6001\u5206\u533a\u8868\uff0c\u5982\u679c\u5176\u5206\u533a\u6570\u975e\u5e38\u591a\uff0c\u90a3\u4e48\u4e5f\u9700\u8981\u6267\u884c\u8f83\u957f\u7684\u65f6\u95f4\u3002<\/li>\n<li>\u5982\u679c\u67e5\u8be2\u7684\u6570\u636e\u4e3a\u5927\u91cf\u7684\u5927\u6587\u4ef6\uff0c\u90a3\u4e48\u5373\u4f7f\u5f00\u542f\u5c0f\u6587\u4ef6\u5408\u5e76\u529f\u80fd\uff0c\u5176\u8f93\u51fa\u6587\u4ef6\u4e5f\u4f9d\u65e7\u5f88\u591a\uff0c\u90a3\u4e48\u4e5f\u9700\u8981\u8f83\u957f\u7684\u65f6\u95f4\u3002<\/li>\n<\/ul>\n<p>12.9.2.5\u00a0\u591a\u5e76\u53d1JDBC\u5ba2\u6237\u7aef\u8fde\u63a5JDBCServer<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>JDBCServer\u652f\u6301\u591a\u7528\u6237\u591a\u5e76\u53d1\u63a5\u5165\uff0c\u4f46\u5f53\u5e76\u53d1\u4efb\u52a1\u6570\u91cf\u8f83\u9ad8\u7684\u65f6\u5019\uff0c\u9ed8\u8ba4\u7684JDBCServer\u914d\u7f6e\u5c06\u65e0\u6cd5\u652f\u6301\uff0c\u56e0\u6b64\u9700\u8981\u8fdb\u884c\u4f18\u5316\u6765\u652f\u6301\u8be5\u573a\u666f\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ol>\n<li>\n<p>\u8bbe\u7f6eJDBCServer\u7684\u516c\u5e73\u8c03\u5ea6\u7b56\u7565\u3002Spark\u9ed8\u8ba4\u4f7f\u7528FIFO\uff08First In First Out\uff09\u7684\u8c03\u5ea6\u7b56\u7565\uff0c\u4f46\u5bf9\u4e8e\u591a\u5e76\u53d1\u7684\u573a\u666f\uff0c\u4f7f\u7528FIFO\u7b56\u7565\u5bb9\u6613\u5bfc\u81f4\u77ed\u4efb\u52a1\u6267\u884c\u5931\u8d25\u3002\u56e0\u6b64\u5728\u591a\u5e76\u53d1\u7684\u573a\u666f\u4e0b\uff0c\u9700\u8981\u4f7f\u7528\u516c\u5e73\u8c03\u5ea6\u7b56\u7565\uff0c\u9632\u6b62\u4efb\u52a1\u6267\u884c\u5931\u8d25\u3002<\/p>\n<\/li>\n<li>\n<p>\u5728Spark\u4e2d\u8bbe\u7f6e\u516c\u5e73\u8c03\u5ea6\uff0c\u5177\u4f53\u8bf7\u53c2\u8003<a href=\"http:\/\/spark.apache.org\/docs\/latest\/job-scheduling.html%5C\">http:\/\/spark.apache.org\/docs\/latest\/job-scheduling.html\\#scheduling-within-an-application<\/a><\/p>\n<ol>\n<li>\u5728JDBC\u5ba2\u6237\u7aef\u4e2d\u8bbe\u7f6e\u516c\u5e73\u8c03\u5ea6\u3002<\/li>\n<\/ol>\n<\/li>\n<li>\n<p>\u5728BeeLine\u547d\u4ee4\u884c\u5ba2\u6237\u7aef\u6216\u8005JDBC\u81ea\u5b9a\u4e49\u4ee3\u7801\u4e2d\uff0c\u6267\u884c\u4ee5\u4e0b\u8bed\u53e5\uff0c<\/p>\n<\/li>\n<\/ol>\n<p>\u5176\u4e2dPoolName\u662f\u516c\u5e73\u8c03\u5ea6\u7684\u67d0\u4e00\u4e2a\u8c03\u5ea6\u6c60\u3002<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2SET spark.sql.thriftserver.scheduler.pool=PoolName;<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/><\/div><\/td><td><div class=\"text codecolorer\">1 &nbsp; &nbsp;1. \u6267\u884c\u76f8\u5e94\u7684SQL\u547d\u4ee4\uff0cSpark\u4efb\u52a1\u5c06\u4f1a\u5728\u4e0a\u9762\u7684\u8c03\u5ea6\u6c60\u4e2d\u8fd0\u884c\u3002 <br \/>\n2<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<ol>\n<li>\u8bbe\u7f6eBroadCastHashJoin\u7684\u7ebf\u7a0b\u6c60\u4e2a\u6570\u3002BroadCastHashJoin\u4f7f\u7528\u591a\u7ebf\u7a0b\u65b9\u5f0f\u5e7f\u64ad\u8868\uff0c\u5728\u591a\u5e76\u53d1\u573a\u666f\u4e0b\uff0c\u4f1a\u6709\u591a\u4e2a\u8868\u540c\u65f6\u5728\u591a\u7ebf\u7a0b\u4e2d\uff0c\u4e00\u65e6\u5e7f\u64ad\u8868\u7684\u4e2a\u6570\u5927\u4e8e\u7ebf\u7a0b\u6c60\u4e2a\u6570\uff0c\u4efb\u52a1\u4f1a\u51fa\u9519\uff0c\u56e0\u6b64\u9700\u8981\u5728JDBCServer\u7684\u201cspark-defaults.conf\u201d\u914d\u7f6e\u6587\u4ef6\u4e2d\u6216\u5728\u547d\u4ee4\u884c\u4e2d\u6267\u884c***set spark.sql.broadcastHashJoin.maxThreadNum=***<em>value<\/em>\uff0c\u8c03\u6574\u7ebf\u7a0b\u6c60\u4e2a\u6570\u3002\u886812-30\u00a0\u53c2\u6570\u63cf\u8ff0<\/li>\n<\/ol>\n<p>spark.sql.broadcastHashJoin.maxThreadNum<br \/>\n\u7528\u4e8eBroadcastHashJoin\u7684\u6700\u5927\u7684\u7ebf\u7a0b\u6c60\u4e2a\u6570\uff0c\u540c\u4e00\u65f6\u95f4\u88ab\u5e7f\u64ad\u7684\u8868\u7684\u4e2a\u6570\u5e94\u8be5\u5c0f\u4e8e\u8be5\u53c2\u6570\u503c\u3002<br \/>\n128<\/p>\n<ol>\n<li>\u8bbe\u7f6eBroadCastHashJoin\u7684\u8d85\u65f6\u65f6\u95f4\u3002BroadCastHashJoin\u6709\u8d85\u65f6\u53c2\u6570\uff0c\u4e00\u65e6\u8d85\u8fc7\u9884\u8bbe\u7684\u65f6\u95f4\uff0c\u8be5\u67e5\u8be2\u4efb\u52a1\u76f4\u63a5\u5931\u8d25\uff0c\u5728\u591a\u5e76\u53d1\u573a\u666f\u4e0b\uff0c\u7531\u4e8e\u8ba1\u7b97\u4efb\u52a1\u62a2\u5360\u8d44\u6e90\uff0c\u53ef\u80fd\u4f1a\u5bfc\u81f4BroadCastHashJoin\u7684Spark\u4efb\u52a1\u65e0\u6cd5\u6267\u884c\uff0c\u5bfc\u81f4\u8d85\u65f6\u51fa\u73b0\u3002\u56e0\u6b64\u9700\u8981\u5728JDBCServer\u7684\u201cspark-defaults.conf\u201d\u914d\u7f6e\u6587\u4ef6\u4e2d\u8c03\u6574\u8d85\u65f6\u65f6\u95f4\u3002\u886812-31\u00a0\u53c2\u6570\u63cf\u8ff0<\/li>\n<\/ol>\n<p>spark.sql.broadcastTimeout<br \/>\nBroadcastHashJoin\u4e2d\u5e7f\u64ad\u8868\u7684\u8d85\u65f6\u65f6\u95f4\uff0c\u5f53\u4efb\u52a1\u5e76\u53d1\u6570\u8f83\u9ad8\u7684\u65f6\u5019\uff0c\u53ef\u4ee5\u8c03\u9ad8\u8be5\u53c2\u6570\u503c\uff0c\u6216\u8005\u76f4\u63a5\u914d\u7f6e\u4e3a\u8d1f\u6570\uff0c\u8d1f\u6570\u4e3a\u65e0\u7a77\u5927\u7684\u8d85\u65f6\u65f6\u95f4\u3002<br \/>\n300\uff08\u6570\u503c\u7c7b\u578b\uff0c\u5b9e\u9645\u4e3a\u4e94\u5206\u949f\uff09<\/p>\n<ol>\n<li>\u8bbe\u7f6e\u4e32\u884cBroadcastHashJoin\u3002\u5f53\u5e76\u53d1\u4efb\u52a1\u975e\u5e38\u91cd\uff08\u4f8b\u5982\u5168\u90e8\u5b8c\u6210\u65f6\u95f4\u8d85\u8fc72\u4e2a\u5c0f\u65f6\uff09\uff0c\u9700\u8981\u5c06BroadcastHashJoin\u8bbe\u7f6e\u4e3a\u4e32\u884c\u5316\uff0c\u8fd9\u6837\u5c31\u80fd\u53bb\u9664\u8d85\u65f6\u65f6\u95f4\u5bf9\u5e76\u53d1\u4efb\u52a1\u7684\u5f71\u54cd\u3002\u4f46\u662f\u4e32\u884c\u5316\u76f8\u5bf9\u4e8e\u5e76\u884c\u5316\uff0c\u4f1a\u964d\u4f4e\u96c6\u7fa4\u8d44\u6e90\u7684\u4f7f\u7528\u7387\uff0c\u56e0\u6b64\u5728\u8f7b\u91cf\u7ea7\u4efb\u52a1\u5e76\u53d1\u65f6\uff0c\u4e0d\u8981\u5f00\u542f\u8be5\u914d\u7f6e\u9879\u3002\u886812-32\u00a0\u53c2\u6570\u63cf\u8ff0<\/li>\n<\/ol>\n<p>spark.sql.bigdata.useSerialBroadcastHashJoin<br \/>\n\u662f\u5426\u4f7f\u7528\u4e32\u884c\u65b9\u5f0f\u6267\u884cBroadcastHashJoin\u3002\u4e32\u884c\u5316BroadcastHashJoin\u4f1a\u964d\u4f4e\u96c6\u7fa4\u8d44\u6e90\u4f7f\u7528\u7387\uff0c\u4f46\u5bf9\u4e8e\u9ad8\u5e76\u53d1\u7684\u91cd\u4efb\u52a1\uff0c\u53ef\u4ee5\u89e3\u51b3\u8d85\u65f6\u7684\u56f0\u6270\u3002<br \/>\nfalse<\/p>\n<p>12.9.2.6\u00a0Parquet\u5143\u6570\u636ebuildScan\u4f18\u5316<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5728\u5206\u533a\u8868\u7684\u573a\u666f\u4e0b\uff0c\u4f1a\u5bf9\u6bcf\u4e2a\u5206\u533a\u4e32\u884c\u6267\u884cbuildScan\u64cd\u4f5c\u6765\u6784\u9020RDD\uff0c\u5728\u6784\u9020RDD\u65f6\u4f1a\u968f\u7740\u5206\u533a\u6570\u7684\u589e\u52a0\u800c\u589e\u52a0\u6267\u884c\u65f6\u95f4\u3002\u56e0\u6b64\uff0c\u63d0\u4f9b\u5e76\u884c\u6267\u884cbuildScan\u64cd\u4f5c\u6765\u6784\u9020RDD\uff0c\u4ece\u800c\u63d0\u5347\u6267\u884c\u6548\u7387\u3002<\/p>\n<p>\u8be5\u4f18\u5316\u4e3b\u8981\u662fDriver\u5229\u7528\u591a\u7ebf\u7a0b\u5e76\u884c\u6267\u884cbuildScan\u64cd\u4f5c\u6765\u63d0\u5347\u6027\u80fd\uff0c\u56e0\u6b64\u4e0d\u9002\u5408\u591asession\u573a\u666f\uff0c\u56e0\u4e3a\u5728\u591asession\u573a\u666f\u4e0b\u6709\u53ef\u80fd\u9020\u6210Driver\u8fd0\u884c\u8fc7\u591a\u7684\u7ebf\u7a0b\uff0c\u4ece\u800c\u9020\u6210\u672a\u77e5\u9519\u8bef\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u5728\u5ba2\u6237\u7aef\u7684\u201cspark-defaults.conf\u201d\u914d\u7f6e\u6587\u4ef6\u4e2d\u8c03\u6574\u5982\u4e0b\u53c2\u6570\u3002<\/p>\n<p>\u886812-33\u00a0\u53c2\u6570\u63cf\u8ff0<\/p>\n<p>spark.sql.sources.parallelBuildScan.threshold<br \/>\n\u5e76\u884c\u6267\u884cbuildScan\u64cd\u4f5c\u7684\u5206\u533a\u6570\u9608\u503c\u3002<br \/>\n-1<br \/>\nspark.sql.sources.parallelBuildScan.threadNum<br \/>\n\u5e76\u884c\u6267\u884cbuildScan\u64cd\u4f5c\u7684\u7ebf\u7a0b\u6570\u3002<br \/>\n2<\/p>\n<p>12.9.2.7\u00a0ParquetRelation InputSplits\u4f18\u5316<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5f53\u524d\u8bfb\u53d6ParquetRelation\u7c7b\u578b\u7684\u6570\u636e\u65f6\u6bcf\u6b21\u90fd\u4f1a\u6267\u884cgetSplits\u64cd\u4f5c\uff0c\u5982\u679c\u8981\u8bfb\u53d6\u7684\u6587\u4ef6\u8f83\u591a\uff0c\u5219\u8017\u65f6\u8f83\u957f\u3002\u56e0\u6b64\u5728\u7b2c\u4e00\u6b21\u6784\u9020Relation\u65f6\u8bfb\u53d6\u5168\u90e8InputSplits\u4fe1\u606f\u5e76\u7f13\u5b58\uff0c\u540e\u7eed\u53ea\u8981\u7f13\u5b58\u6ca1\u88ab\u6e05\u9664\uff0c\u5219\u6bcf\u6b21\u53ea\u9700\u4ece\u7f13\u5b58\u4e2d\u8bfb\u53d6\u6240\u9700\u7684InputSplits\u4fe1\u606f\uff0c\u4ece\u800c\u63d0\u5347\u975e\u7b2c\u4e00\u6b21\u67e5\u8be2\u7684\u6027\u80fd\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u5728\u5ba2\u6237\u7aef\u7684\u201cspark-defaults.conf\u201d\u914d\u7f6e\u6587\u4ef6\u8c03\u6574\u5982\u4e0b\u53c2\u6570\u3002<\/p>\n<p>\u886812-34\u00a0\u53c2\u6570\u63cf\u8ff0<\/p>\n<p>spark.sql.source.inputSplit.useCache<br \/>\n\u662f\u5426\u5f00\u542f\u7f13\u5b58ParquetRelation\u7684InputSplits\u4fe1\u606f\u529f\u80fd\u3002<br \/>\nfalse<\/p>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<ul>\n<li>\u53ea\u6709\u5728\u6570\u636e\u6587\u4ef6\u6570\u5728\u533a\u95f4(1000, 100000]\u4e4b\u5185\uff0c\u4e14spark.sql.source.inputSplit.useCache=true\u65f6\u624d\u4f1a\u7f13\u5b58ParquetRelation\u7684InputSplits\u4fe1\u606f\u3002<\/li>\n<li>\u7f13\u5b58ParquetRelation\u7684InputSplits\u4fe1\u606f\u529f\u80fd\u53ea\u4f1a\u5728\u7b2c\u4e00\u6b21\u6784\u9020Relation\u65f6\uff08\u4f8b\u5982\u7b2c\u4e00\u6b21\u6267\u884c\u67e5\u8be2\u64cd\u4f5c\u65f6\uff09\u751f\u6548\u3002\u5982\u679c\u5728\u7b2c\u4e00\u6b21\u6784\u9020Relation\u4e4b\u540e\u624d\u5f00\u542f\u7f13\u5b58ParquetRelation\u7684InputSplits\u4fe1\u606f\u529f\u80fd\uff0c\u90a3\u4e48\u53ef\u901a\u8fc7\u6267\u884cREFRESH\u64cd\u4f5c\u6765\u6e05\u9664\u5bf9\u5e94\u7684Relation\u7f13\u5b58\u3002<\/li>\n<li>\u4e00\u65e6\u6267\u884c\u8fc7\u6570\u636e\u5bfc\u5165\uff0c\u5fc5\u987b\u6267\u884cREFRESH\u64cd\u4f5c\u6765\u5237\u65b0\u7f13\u5b58\uff0c\u5426\u5219\u6709\u53ef\u80fd\u9020\u6210\u6267\u884c\u67e5\u8be2\u64cd\u4f5c\u65f6\u4e0a\u62a5\u6587\u4ef6\u4e0d\u5b58\u5728\u5f02\u5e38\u3002<\/li>\n<\/ul>\n<p>12.9.2.8\u00a0Limit\u4f18\u5316<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5728RDD\u7684Partition\u6570\u8fc7\u591a\u65f6\uff08\u5927\u4e8e\u6216\u7b49\u4e8e1000\uff0c\u4f8b\u5982spark.sql.shuffle.partitions=2000\u65f6\uff09\uff0c\u5efa\u8bae\u4f7f\u7528Limit\u4f18\u5316\uff0c\u51cf\u5c11\u626b\u63cf\u7684\u6587\u4ef6\u6570\u3002<\/p>\n<p>\u5728\u7269\u7406\u8ba1\u5212\u4e2d\uff0cLimit\u5206\u4e3a\u975e\u672b\u7aefLimit\u548c\u672b\u7aefLimit\u3002\u975e\u672b\u7aefLimit\u4f18\u5316\u53ef\u4ee5\u901a\u8fc7\u201cspark.sql.optimize.limit\u201d\u914d\u7f6e\u9879\u8fdb\u884c\u5f00\u542f\u6216\u5173\u95ed\u3002\u672b\u7aefLimit\u4f18\u5316\u6ca1\u6709\u63a7\u5236\u5f00\u5173\uff0c\u9ed8\u8ba4\u90fd\u8fdb\u884c\u4f18\u5316\u3002\u7528\u6237\u53ef\u4ee5\u901a\u8fc7\u5728Spark SQL\u5ba2\u6237\u7aef\u4e2d\u6267\u884c***explain <!-- -->&lt;SQL statement&gt;<!-- -->***\u547d\u4ee4\u67e5\u770b\u7269\u7406\u8ba1\u5212\u3002<\/p>\n<p>\u4e3e\u4f8b\u8bf4\u660e\uff1a<\/p>\n<p>\u5982\u4e0b\u6240\u793a\uff0c\u4f7f\u7528<em><strong>explain<\/strong><\/em>\u547d\u4ee4\u67e5\u8be2\u7269\u7406\u8ba1\u5212\uff0c\u5728\u67e5\u8be2\u7ed3\u679c\u7684\u4e2d\u95f4\u4f4d\u7f6e\u663e\u793a\u7684\u5c31\u662f\u975e\u672b\u7aefLimit\uff0c\u5373\u4e0b\u9762\u84dd\u8272\u5b57\u4f53\u6807\u8bc6\u7684\u90e8\u5206\u3002<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/>6<br \/>7<br \/>8<br \/>9<br \/>10<br \/>11<br \/>12<br \/>13<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2spark-sql&amp;gt; explain select count(*) from (select key, sum(value) from src group by key limit 10) t;<br \/>\n3== Physical Plan ==<br \/>\n4TungstenAggregate(key=[], functions=[(count(1),mode=Final,isDistinct=false)], output=[_c0#24L])<br \/>\n5TungstenAggregate(key=[], functions=[(count(1),mode=Partial,isDistinct=false)], output=[currentCount#27L])<br \/>\n6Limit 10<br \/>\n7ConvertToSafe<br \/>\n8TungstenAggregate(key=[key#21], functions=[], output=[])<br \/>\n9TungstenExchange hashpartitioning(key#21)<br \/>\n10TungstenAggregate(key=[key#21], functions=[], output=[key#21])<br \/>\n11HiveTableScan [key#21], (MetastoreRelation default, src, None), Statistics(5812)<br \/>\n12Time taken: 0.119 seconds, Fetched 9 row(s)<br \/>\n13<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u5982\u4e0b\u6240\u793a\uff0c\u4f7f\u7528<em><strong>explain<\/strong><\/em>\u547d\u4ee4\u67e5\u8be2\u7269\u7406\u8ba1\u5212\uff0c\u5728\u67e5\u8be2\u7ed3\u679c\u7684\u6700\u524d\u9762\u663e\u793a\u7684\u5c31\u662f\u672b\u7aefLimit\uff0c\u5373\u4e0b\u9762\u84dd\u8272\u5b57\u4f53\u6807\u8bc6\u7684\u90e8\u5206\u3002<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/>4<br \/>5<br \/>6<br \/>7<br \/>8<br \/>9<br \/>10<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2spark-sql&amp;gt; explain select key, sum(value) from src group by key limit 10;<br \/>\n3== Physical Plan ==<br \/>\n4Limit 10<br \/>\n5ConvertToSafe<br \/>\n6TungstenAggregate(key=[key#33], functions=[(sum(cast(value#34 as double)),mode=Final,isDistinct=false)], output=[key#33,_c1#35])<br \/>\n7TungstenExchange hashpartitioning(key#33)<br \/>\n8TungstenAggregate(key=[key#33], functions=[(sum(cast(value#34 as double)),mode=Partial,isDistinct=false)], output=[key#33,currentSum#38])<br \/>\n9HiveTableScan [key#33,value#34], (MetastoreRelation default, src, None), Statistics(5812)<br \/>\n10<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u5728\u5ba2\u6237\u7aef\u7684\u201cspark-defaults.conf\u201d\u914d\u7f6e\u6587\u4ef6\u4e2d\u8c03\u6574\u5982\u4e0b\u53c2\u6570\u3002<\/p>\n<p>\u886812-35\u00a0\u53c2\u6570\u63cf\u8ff0<\/p>\n<p>spark.sql.optimize.limit<br \/>\n\u975e\u672b\u7aefLimit\u4f18\u5316\u5f00\u5173\u3002\u5f00\u542f\u672c\u5f00\u5173\uff0c\u4e14Limit\u4e2a\u6570\u5c0f\u4e8e\u7b49\u4e8e1000\u65f6\uff0c\u4f18\u5316\u751f\u6548\uff0c\u4f7f\u975e\u672b\u7aefLimit\u4e5f\u4f7f\u7528\u672b\u7aefLimit\u7684\u5206\u6279\u8bfb\u53d6\u7684\u7b97\u6cd5\u3002\u5efa\u8bae\u5f00\u542f\u3002<br \/>\nfalse<br \/>\nspark.sql.limit.numPartsToTry.first<br \/>\n\u5728Limit\u573a\u666f\u4e0b\u7b2c\u4e00\u6b21\u8bfb\u53d6\u7684Partition\u6570\u3002\u5728RDD\u7684Partition\u6570\u8fc7\u591a\u65f6\uff0c\u5efa\u8bae\u5f00\u542f\u3002\u5efa\u8bae\u914d\u7f6e\u533a\u95f4\u4e3a[1, min(limit value, 10)]\u3002<br \/>\n-1<br \/>\nspark.sql.limit.numPartsToTry.other<br \/>\n\u5728Limit\u573a\u666f\u4e0b\u540e\u7eed\u8bfb\u53d6\u7684Partition\u6570\u3002\u5728RDD\u7684Partition\u6570\u8fc7\u591a\u65f6\uff0c\u5efa\u8bae\u5f00\u542f\u3002\u5efa\u8bae\u914d\u7f6e\u533a\u95f4\u4e3a[50, 100]\u3002\u751f\u6548\u6761\u4ef6\u662fspark.sql.limit.numPartsToTry.first&gt;0\uff0c\u82e5\u4e0d\u914d\u7f6e\u5219\u9ed8\u8ba4\u4f7f\u7528spark.sql.limit.numPartsToTry.first\u7684\u53c2\u6570\u503c\u3002<br \/>\n-1<\/p>\n<p>12.9.2.9\u00a0LimitScan\u4f18\u5316<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u7528\u6237\u573a\u666f\u4e2d\uff0c\u5b58\u5728\u5feb\u901f\u5c55\u793a\u6570\u636e\u7684\u573a\u666f\uff0c\u5178\u578b\u7684SQL\u8bed\u53e5\u5982\u4e0b\u6240\u793a\uff1a<\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2select col1, col2 from table limit 100;<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p>\u901a\u8fc7\u6267\u884c\u5305\u542bLimit\u5b50\u53e5\u7684Spark SQL\u8bed\u53e5\uff0c\u6765\u67e5\u8be2\u8868\u7684\u90e8\u5206\u6570\u636e\uff0c\u5e76\u5feb\u901f\u5c55\u793a\uff0c\u4f46\u662f\u5728\u67e5\u8be2parquet\u5927\u8868\u65f6\uff0c\u7531\u4e8e\u83b7\u53d6\u5927\u8868\u5206\u533a\u4fe1\u606f\u8017\u65f6\u8f83\u957f\uff0c\u5bfc\u81f4\u65e0\u6cd5\u8fbe\u5230\u5b9e\u65f6\u67e5\u8be2\u7684\u76ee\u7684\u3002\u6b64\u65f6\u7528\u6237\u53ef\u4ee5\u5f00\u542fLimitScan\u4f18\u5316\uff0c\u63d0\u9ad8\u67e5\u8be2\u6027\u80fd\uff0c\u5feb\u901f\u5c55\u793a\u67e5\u8be2\u7ed3\u679c\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u5728\u5ba2\u6237\u7aef\u7684\u201cspark-defaults.conf\u201d\u914d\u7f6e\u6587\u4ef6\u4e2d\u8c03\u6574\u5982\u4e0b\u53c2\u6570\u3002<\/p>\n<p>\u886812-36\u00a0\u53c2\u6570\u63cf\u8ff0<\/p>\n<p>spark.sql.limitScan.enabled<br \/>\n\u662f\u5426\u5f00\u542fLimitScan\u4f18\u5316\u3002<br \/>\ntrue<br \/>\nspark.sql.limitScan.num<br \/>\nLimit\u4e2a\u6570\u9650\u5236\uff0c\u8d85\u8fc7\u8fd9\u4e2a\u6570\u503c\uff0c\u5219\u4e0d\u505aLimitScan\u4f18\u5316\u3002<br \/>\n5000<\/p>\n<p>\u4f7f\u7528\u7ea6\u675f<\/p>\n<ul>\n<li>\u53ea\u9002\u7528\u4e8eHive\u547d\u4ee4\u5efa\u7684\u8868\uff0c\u4e0d\u80fd\u7528\u4e8eDataSource\u8868\u3002<\/li>\n<li>Select\u7684\u8868\u5b57\u6bb5\u7c7b\u578b\u53ea\u9002\u7528\u4e8e\u539f\u751f\u6570\u636e\u7c7b\u578b\u3002\u5982\u679c\u6709\u590d\u6742\u6570\u636e\u7c7b\u578b\uff08Array\uff0cStruct\uff0cMap\uff09\uff0c\u5219\u4e0d\u505aLimitScan\u4f18\u5316\u3002<\/li>\n<li>Select\u7684\u5b57\u6bb5\u5fc5\u987b\u4e3a\u8868\u4e2d\u771f\u5b9e\u5b57\u6bb5\uff0c\u6216\u8005\u8be5\u5b57\u6bb5\u7684\u522b\u540d\uff0c\u4f8b\u5982\u201cselect col1, col2 as alias\u201d\u3002<\/li>\n<li>Select\u5b57\u6bb5\u4e2d\u5982\u679c\u6709\u5e26\u7b97\u672f\u8fd0\u7b97\u3001\u51fd\u6570\u7684\u5b57\u6bb5\uff0c\u5219\u4e0d\u652f\u6301LimitScan\u4f18\u5316\uff0c\u4f8b\u5982\u201cselect col1*100+100, avg(col2) from table1 limit 100\u201d\u3002<\/li>\n<li>\u4e0d\u9002\u7528\u4e8e\u5305\u542bWHERE\u3001HAVING\u3001GROUP BY\u3001CLUSTER BY\u3001DISTRIBUTE BY\u3001SORT BY\u3001ORDER BY\u5b50\u53e5\u7684Select\u8bed\u53e5\u3002<\/li>\n<li>\u82e5Spark\u670d\u52a1\u7aef\u7684\u201chive-site.xml\u201d\u6587\u4ef6\u4e2d\u201chive.server2.enable.doAs\u201d\u914d\u7f6e\u6210\u201ctrue\u201d\uff0c\u5219\u65e0\u6cd5\u4f7f\u7528LimitScan\u529f\u80fd\u3002<\/li>\n<\/ul>\n<p>12.9.2.10\u00a0\u9884\u5148Broadcast\u5c0f\u8868\u4f18\u5316<\/p>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5f00\u542f\u9884\u5148Broadcast\u5c0f\u8868\u4f18\u5316\u540e\uff0c\u540c\u4e00SQL\u8bed\u53e5\u6216\u4e0d\u540cSQL\u8bed\u53e5\u5185\uff0c\u5b58\u5728\u76f8\u540c\u7684\u5c0f\u8868\u6216\u5bf9\u76f8\u540c\u5c0f\u8868\u505a\u5b50\u67e5\u8be2\u65f6\uff0c\u53ea\u9700\u8981\u5c06\u5c0f\u8868\u5e7f\u64ad\u4e00\u6b21\uff0c\u540e\u7eed\u5c31\u53ef\u4ee5\u590d\u7528\u7f13\u5b58\u5728\u5185\u5b58\u4e2d\u7684\u8868\u6570\u636e\uff0c\u907f\u514d\u91cd\u590d\u5e7f\u64ad\uff0c\u4ece\u800c\u63d0\u5347SQL\u7684\u6027\u80fd\u3002<\/p>\n<p>\u652f\u6301\u9884\u5e7f\u64ad\u7684\u5c0f\u8868\u662f\u6307\u5c0f\u4e8e\u81ea\u52a8\u5e7f\u64ad\u9608\u503c\u7684\u8868\uff0c\u7528\u6237\u53ef\u4ee5\u901a\u8fc7Spark\u7684\u914d\u7f6e\u6587\u4ef6\u201cspark-defaults.conf\u201d\u4e2d\u7684\u201cspark.sql.autoBroadcastJoinThreshold\u201d\u53c2\u6570\u6307\u5b9a\u81ea\u52a8\u5e7f\u64ad\u9608\u503c\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<ol>\n<li>\u901a\u8fc7\u5982\u4e0b\u914d\u7f6e\u9879\u5f00\u542f\u9884\u5e7f\u64ad\u5c0f\u8868\u7684\u529f\u80fd\u3002\u5728\u5ba2\u6237\u7aef\u7684\u201cspark-defaults.conf\u201d\u914d\u7f6e\u6587\u4ef6\u4e2d\u8c03\u6574\u5982\u4e0b\u53c2\u6570\u3002\u886812-37\u00a0\u53c2\u6570\u63cf\u8ff0<\/li>\n<\/ol>\n<p>spark.sql.saveBroadcastTables.enabled<br \/>\n\u662f\u5426\u5f00\u542f\u9884\u5148\u5e7f\u64ad\u5c0f\u8868\u7684\u4f18\u5316\u529f\u80fd\u3002<br \/>\ntrue \u5728\u670d\u52a1\u7aef\u7684\u201cspark-defaults.conf\u201d\u914d\u7f6e\u6587\u4ef6\u4e2d\u8c03\u6574\u5982\u4e0b\u53c2\u6570\u3002\u91cd\u542f\u670d\u52a1\u7aef\u540e\u624d\u751f\u6548\u3002\u886812-38\u00a0\u53c2\u6570\u63cf\u8ff0<\/p>\n<p>spark.sql.broadcastTables.sizeInBytes<br \/>\nDriver\u4e2d\u7f13\u5b58\u7684\u8868\u5927\u5c0f\uff08\u8868\u5728HDFS\u4e0a\u7684\u5927\u5c0f\uff0c\u800c\u4e0d\u662f\u8868\u5728\u5185\u5b58\u4e2d\u7f13\u5b58\u7684\u5b9e\u9645\u5185\u5bb9\u7684\u5927\u5c0f\uff09\u7684\u9608\u503c\uff0c\u8d85\u8fc7\u8fd9\u4e2a\u9608\u503c\u5219\u91c7\u7528FIFO\u7684\u65b9\u5f0f\u6324\u51fa\u6700\u5148\u7f13\u5b58\u7684\u8868\u3002\u5355\u4f4d\u4e3aByte\u3002 \u8bf4\u660e\uff1a \u8fd9\u662f\u670d\u52a1\u7aef\u53c2\u6570\uff0c\u4e0d\u80fd\u5728\u5ba2\u6237\u7aef\u901a\u8fc7SET\u7684\u65b9\u5f0f\u8fdb\u884c\u8bbe\u7f6e\u3002<br \/>\n104857600<\/p>\n<ol>\n<li>\uff08\u53ef\u9009\uff09\u76f8\u5173\u547d\u4ee4\u4ecb\u7ecd\u3002<\/li>\n<\/ol>\n<ul>\n<li>\n<p>\u5982\u679c\u7528\u6237\u66f4\u65b0\u5206\u533a\u5c0f\u8868\uff0c\u66f4\u65b0\u540e\u9700\u8981\u624b\u52a8\u6267\u884c<em><strong>Analyze<\/strong><\/em>\u547d\u4ee4\uff08Text\u8868\uff09\u6216<em><strong>Refresh<\/strong><\/em>\u547d\u4ee4\uff08Parquet\u8868\u6216Orc\u8868\uff09\u6765\u66f4\u65b0\u5143\u6570\u636e\uff0c\u4ece\u800c\u91cd\u65b0\u7f13\u5b58\u66f4\u65b0\u540e\u7684\u5206\u533a\u5c0f\u8868\u3002<\/p>\n<\/li>\n<li>\n<p>Analyze\u547d\u4ee4<\/p>\n<\/li>\n<\/ul>\n<p><em><strong>ANALYZE TABLE tableName COMPUTE STATISTICS noscan;<\/strong><\/em><\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/><\/div><\/td><td><div class=\"text codecolorer\">1 &nbsp; &nbsp;* Refresh\u547d\u4ee4 <br \/>\n2<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p><em><strong>Refresh table tableName;<\/strong><\/em><\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/><\/div><\/td><td><div class=\"text codecolorer\">1* \u663e\u793a\u6240\u6709\u7f13\u5b58\u7684\u5e7f\u64ad\u8868\u3002 <br \/>\n2<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p><em><strong>SHOW BROADCAST TABLES<\/strong><\/em><\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/><\/div><\/td><td><div class=\"text codecolorer\">1* \u6e05\u9664\u6240\u6709\u7f13\u5b58\u7684\u5e7f\u64ad\u8868\u3002 <br \/>\n2<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n<p><em><strong>CLEAR BROADCAST<\/strong><\/em><\/p>\n<p>\u53ea\u6709\u6388\u4e88\u201cADMIN\u201d\u6743\u9650\u7684\u7528\u6237\u624d\u80fd\u6267\u884c\u8be5\u547d\u4ee4\u3002<\/p>\n<p>\u6267\u884c\u8be5\u547d\u4ee4\u540e\uff0c\u518d\u6b21\u8fd0\u884cSQL\u8bed\u53e5\u65f6\uff0c\u4f1a\u518d\u89e6\u53d1\u5e7f\u64ad\u5e76\u5c06\u5e7f\u64ad\u8868\u7f13\u5b58\u8d77\u6765\u3002<\/p>\n<p>\u4f7f\u7528\u9650\u5236<\/p>\n<ul>\n<li>\u4e0d\u652f\u6301Cache\u5728\u5185\u5b58\u4e2d\u7684\u5c0f\u8868\u3002<\/li>\n<li>\u4e0d\u652f\u6301\u624b\u52a8\u4fee\u6539\u8868\u7684totalSize\u548cModifyTime\u7b49\u5143\u6570\u636e\u4fe1\u606f\u3002<\/li>\n<li>\u652f\u6301\u9884\u5e7f\u64ad\u7684\u5c0f\u8868\u683c\u5f0f\u4e3aText\uff0cParquet\u548cOrc\u3002<\/li>\n<li>\u652f\u6301\u7684Join\u7c7b\u578b\u6709\uff1aInner Join\u3001Left Join\u3001Right Join\u3001Full Join\u3002<\/li>\n<li>spark-sql\u4e2d\u7f13\u5b58\u7684\u5e7f\u64ad\u8868\u53ea\u652f\u6301\u5728\u5355\u4e2aCLI\u5185\u5171\u4eab\uff08spark-sql\u91cd\u542f\u540e\u9700\u8981\u91cd\u65b0\u5e7f\u64ad\uff0c\u89e6\u53d1\u7f13\u5b58\uff09\u3002<\/li>\n<\/ul>\n<h3>12.9.3\u00a0Spark Streaming\u8c03\u4f18<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>Spark\u00a0Streaming\u4f5c\u4e3a\u4e00\u79cdmini-batch\u65b9\u5f0f\u7684\u6d41\u5f0f\u5904\u7406\u6846\u67b6\uff0c\u5b83\u4e3b\u8981\u7684\u7279\u70b9\u662f\uff1a\u79d2\u7ea7\u65f6\u5ef6\u548c\u9ad8\u541e\u5410\u91cf\u3002\u56e0\u6b64Spark\u00a0Streaming\u8c03\u4f18\u7684\u76ee\u6807\uff1a\u5728\u79d2\u7ea7\u5ef6\u8fdf\u7684\u60c5\u666f\u4e0b\uff0c\u63d0\u9ad8Spark\u00a0Streaming\u7684\u541e\u5410\u80fd\u529b\uff0c\u5728\u5355\u4f4d\u65f6\u95f4\u5904\u7406\u5c3d\u53ef\u80fd\u591a\u7684\u6570\u636e\u3002<\/p>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u672c\u7ae0\u8282\u9002\u7528\u4e8e\u8f93\u5165\u6570\u636e\u6e90\u4e3aKafka\u7684\u4f7f\u7528\u573a\u666f\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u4e00\u4e2a\u7b80\u5355\u7684\u6d41\u5904\u7406\u7cfb\u7edf\u7531\u4ee5\u4e0b\u4e09\u90e8\u5206\u7ec4\u4ef6\u7ec4\u6210\uff1a\u6570\u636e\u6e90 + \u63a5\u6536\u5668 + \u5904\u7406\u5668\u3002\u6570\u636e\u6e90\u4e3aKafka\uff0c\u63a5\u53d7\u5668\u4e3aSpark\u00a0Streaming\u4e2d\u7684Kafka\u6570\u636e\u6e90\u63a5\u6536\u5668\uff0c\u5904\u7406\u5668\u4e3aSpark\u00a0Streaming\u3002<\/p>\n<p>\u5bf9Spark\u00a0Streaming\u8c03\u4f18\uff0c\u5c31\u5fc5\u987b\u4f7f\u8be5\u4e09\u4e2a\u90e8\u4ef6\u7684\u6027\u80fd\u90fd\u6700\u4f18\u5316\u3002<\/p>\n<ul>\n<li><strong>\u6570\u636e\u6e90\u8c03\u4f18<\/strong><\/li>\n<\/ul>\n<p>\u5728\u5b9e\u9645\u7684\u5e94\u7528\u573a\u666f\u4e2d\uff0c\u6570\u636e\u6e90\u4e3a\u4e86\u4fdd\u8bc1\u6570\u636e\u7684\u5bb9\u9519\u6027\uff0c\u4f1a\u5c06\u6570\u636e\u4fdd\u5b58\u5728\u672c\u5730\u78c1\u76d8\u4e2d\uff0c\u800cSpark\u00a0Streaming\u7684\u8ba1\u7b97\u7ed3\u679c\u5f80\u5f80\u5168\u90e8\u5728\u5185\u5b58\u4e2d\u5b8c\u6210\uff0c\u6570\u636e\u6e90\u5f88\u6709\u53ef\u80fd\u6210\u4e3a\u6d41\u5f0f\u7cfb\u7edf\u7684\u6700\u5927\u74f6\u9888\u70b9\u3002<\/p>\n<p>\u5bf9Kafka\u7684\u6027\u80fd\u8c03\u4f18\uff0c\u6709\u4ee5\u4e0b\u51e0\u4e2a\u70b9\uff1a<\/p>\n<ul>\n<li>\u4f7f\u7528Kafka-0.8.2\u4ee5\u540e\u7248\u672c\uff0c\u53ef\u4ee5\u4f7f\u7528\u5f02\u6b65\u6a21\u5f0f\u7684\u65b0Producer\u63a5\u53e3\u3002<!-- -->\n<ul>\n<li>\u914d\u7f6e\u591a\u4e2aBroker\u7684\u76ee\u5f55\uff0c\u8bbe\u7f6e\u591a\u4e2aIO\u7ebf\u7a0b\uff0c\u914d\u7f6eTopic\u5408\u7406\u7684Partition\u4e2a\u6570\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u8be6\u60c5\u8bf7\u53c2\u89c1Kafka\u5f00\u6e90\u6587\u6863\u4e2d\u7684\u201c\u6027\u80fd\u8c03\u4f18\u201d\u90e8\u5206\uff1a<a href=\"http:\/\/kafka.apache.org\/documentation.html\">http:\/\/kafka.apache.org\/documentation.html<\/a><\/p>\n<ul>\n<li><strong>\u63a5\u6536\u5668\u8c03\u4f18<\/strong><\/li>\n<\/ul>\n<p>Spark\u00a0Streaming\u4e2d\u5df2\u6709\u591a\u79cd\u6570\u636e\u6e90\u7684\u63a5\u6536\u5668\uff0c\u4f8b\u5982Kafka\u3001Flume\u3001MQTT\u3001ZeroMQ\u7b49\uff0c\u5176\u4e2dKafka\u7684\u63a5\u6536\u5668\u7c7b\u578b\u6700\u591a\uff0c\u4e5f\u662f\u6700\u6210\u719f\u4e00\u5957\u63a5\u6536\u5668\u3002<\/p>\n<p>Kafka\u5305\u62ec\u4e09\u79cd\u6a21\u5f0f\u7684\u63a5\u6536\u5668API\uff1a<\/p>\n<ul>\n<li>KafkaReceiver\uff1a\u76f4\u63a5\u63a5\u6536Kafka\u6570\u636e\uff0c\u8fdb\u7a0b\u5f02\u5e38\u540e\uff0c\u53ef\u80fd\u51fa\u73b0\u6570\u636e\u4e22\u5931\u3002<!-- -->\n<ul>\n<li>ReliableKafkaReceiver\uff1a\u901a\u8fc7ZooKeeper\u8bb0\u5f55\u63a5\u6536\u6570\u636e\u4f4d\u79fb\u3002<\/li>\n<li>DirectKafka\uff1a\u76f4\u63a5\u901a\u8fc7RDD\u8bfb\u53d6Kafka\u6bcf\u4e2aPartition\u4e2d\u7684\u6570\u636e\uff0c\u6570\u636e\u9ad8\u53ef\u9760\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u4ece\u5b9e\u73b0\u4e0a\u6765\u770b\uff0cDirectKafka\u7684\u6027\u80fd\u4f1a\u662f\u6700\u597d\u7684\uff0c\u5b9e\u9645\u6d4b\u8bd5\u4e0a\u6765\u770b\uff0cDirectKafka\u4e5f\u786e\u5b9e\u6bd4\u5176\u4ed6\u4e24\u4e2aAPI\u6027\u80fd\u597d\u4e86\u4e0d\u5c11\u3002\u56e0\u6b64\u63a8\u8350\u4f7f\u7528DirectKafka\u7684API\u5b9e\u73b0\u63a5\u6536\u5668\u3002<\/p>\n<p>\u6570\u636e\u63a5\u6536\u5668\u4f5c\u4e3a\u4e00\u4e2aKafka\u7684\u6d88\u8d39\u8005\uff0c\u5bf9\u4e8e\u5b83\u7684\u914d\u7f6e\u4f18\u5316\uff0c\u8bf7\u53c2\u89c1Kafka\u5f00\u6e90\u6587\u6863\uff1a<a href=\"http:\/\/kafka.apache.org\/documentation.html\">http:\/\/kafka.apache.org\/documentation.html<\/a><\/p>\n<ul>\n<li><strong>\u5904\u7406\u5668\u8c03\u4f18<\/strong><\/li>\n<\/ul>\n<p>Spark\u00a0Streaming\u7684\u5e95\u5c42\u7531Spark\u6267\u884c\uff0c\u56e0\u6b64\u5927\u90e8\u5206\u5bf9\u4e8eSpark\u7684\u8c03\u4f18\u63aa\u65bd\uff0c\u90fd\u53ef\u4ee5\u5e94\u7528\u5728Spark\u00a0Streaming\u4e4b\u4e2d\uff0c\u4f8b\u5982\uff1a<\/p>\n<ul>\n<li>\u6570\u636e\u5e8f\u5217\u5316<!-- -->\n<ul>\n<li>\u914d\u7f6e\u5185\u5b58<\/li>\n<li>\u8bbe\u7f6e\u5e76\u884c\u5ea6<\/li>\n<li>\u4f7f\u7528External Shuffle Service\u63d0\u5347\u6027\u80fd<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u5728\u505aSpark Streaming\u7684\u6027\u80fd\u4f18\u5316\u65f6\u9700\u6ce8\u610f\u4e00\u70b9\uff0c\u8d8a\u8ffd\u6c42\u6027\u80fd\u4e0a\u7684\u4f18\u5316\uff0cSpark\u00a0Streaming\u6574\u4f53\u7684\u53ef\u9760\u6027\u4f1a\u8d8a\u5dee\u3002\u4f8b\u5982\uff1a<\/p>\n<p>\u201cspark.streaming.receiver.writeAheadLog.enable\u201d\u914d\u7f6e\u4e3a\u201cfalse\u201d\u7684\u65f6\u5019\uff0c\u4f1a\u660e\u663e\u51cf\u5c11\u78c1\u76d8\u7684\u64cd\u4f5c\uff0c\u63d0\u9ad8\u6027\u80fd\uff0c\u4f46\u7531\u4e8e\u7f3a\u5c11WAL\u673a\u5236\uff0c\u4f1a\u51fa\u73b0\u5f02\u5e38\u6062\u590d\u65f6\uff0c\u6570\u636e\u4e22\u5931\u3002<\/p>\n<p>\u56e0\u6b64\uff0c\u5728\u8c03\u4f18Spark\u00a0Streaming\u7684\u65f6\u5019\uff0c\u8fd9\u4e9b\u4fdd\u8bc1\u6570\u636e\u53ef\u9760\u6027\u7684\u914d\u7f6e\u9879\uff0c\u5728\u751f\u4ea7\u73af\u5883\u4e2d\u662f\u4e0d\u80fd\u5173\u95ed\u7684\u3002<\/p>\n<h3>12.9.4\u00a0Spark CBO\u8c03\u4f18<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>SQL\u8bed\u53e5\u8f6c\u5316\u4e3a\u5177\u4f53\u6267\u884c\u8ba1\u5212\u662f\u7531SQL\u67e5\u8be2\u7f16\u8bd1\u5668\u51b3\u5b9a\u7684\uff0c\u540c\u4e00\u4e2aSQL\u8bed\u53e5\u53ef\u4ee5\u8f6c\u5316\u6210\u591a\u79cd\u7269\u7406\u6267\u884c\u8ba1\u5212\uff0c\u5982\u4f55\u6307\u5bfc\u7f16\u8bd1\u5668\u9009\u62e9\u6548\u7387\u6700\u9ad8\u7684\u6267\u884c\u8ba1\u5212\uff0c\u8fd9\u5c31\u662f\u4f18\u5316\u5668\u7684\u4e3b\u8981\u4f5c\u7528\u3002\u4f20\u7edf\u6570\u636e\u5e93\uff08\u4f8b\u5982Oracle\uff09\u7684\u4f18\u5316\u5668\u6709\u4e24\u79cd\uff1a\u57fa\u4e8e\u89c4\u5219\u7684\u4f18\u5316\u5668(Rule-Based Optimization\uff0cRBO)\u548c\u57fa\u4e8e\u4ee3\u4ef7\u7684\u4f18\u5316\u5668(Cost-Based Optimization\uff0cCBO)\u3002<\/p>\n<ul>\n<li>RBO<\/li>\n<\/ul>\n<p>RBO\u4f7f\u7528\u7684\u89c4\u5219\u662f\u6839\u636e\u7ecf\u9a8c\u5f62\u6210\u7684\uff0c\u53ea\u8981\u6309\u7167\u8fd9\u4e2a\u89c4\u5219\u53bb\u5199SQL\u8bed\u53e5\uff0c\u65e0\u8bba\u6570\u636e\u8868\u4e2d\u7684\u5185\u5bb9\u600e\u6837\u3001\u6570\u636e\u5206\u5e03\u5982\u4f55\uff0c\u90fd\u4e0d\u4f1a\u5f71\u54cd\u5230\u6267\u884c\u8ba1\u5212\u3002<\/p>\n<ul>\n<li>CBO<\/li>\n<\/ul>\n<p>CBO\u662f\u6839\u636e\u5b9e\u9645\u6570\u636e\u5206\u5e03\u548c\u7ec4\u7ec7\u60c5\u51b5\uff0c\u8bc4\u4f30\u6bcf\u4e2a\u8ba1\u5212\u7684\u6267\u884c\u4ee3\u4ef7\uff0c\u4ece\u800c\u9009\u62e9\u4ee3\u4ef7\u6700\u5c0f\u7684\u6267\u884c\u8ba1\u5212\u3002<\/p>\n<p>\u76ee\u524dSpark\u7684\u4f18\u5316\u5668\u90fd\u662f\u57fa\u4e8eRBO\u7684\uff0c\u5df2\u7ecf\u6709\u6570\u5341\u6761\u4f18\u5316\u89c4\u5219\uff0c\u4f8b\u5982\u8c13\u8bcd\u4e0b\u63a8\u3001\u5e38\u91cf\u6298\u53e0\u3001\u6295\u5f71\u88c1\u526a\u7b49\uff0c\u8fd9\u4e9b\u89c4\u5219\u662f\u6709\u6548\u7684\uff0c\u4f46\u662f\u5b83\u5bf9\u6570\u636e\u662f\u4e0d\u654f\u611f\u7684\u3002\u5bfc\u81f4\u7684\u4e00\u4e2a\u95ee\u9898\u5c31\u662f\u6570\u636e\u8868\u4e2d\u6570\u636e\u5206\u5e03\u53d1\u751f\u53d8\u5316\u65f6\uff0cRBO\u662f\u4e0d\u611f\u77e5\u7684\uff0c\u57fa\u4e8eRBO\u751f\u6210\u7684\u6267\u884c\u8ba1\u5212\u4e0d\u80fd\u786e\u4fdd\u662f\u6700\u4f18\u7684\u3002\u800cCBO\u7684\u91cd\u8981\u4f5c\u7528\u5c31\u662f\u80fd\u591f\u6839\u636e\u5b9e\u9645\u6570\u636e\u5206\u5e03\u4f30\u7b97\u51faSQL\u8bed\u53e5\uff0c\u751f\u6210\u4e00\u7ec4\u53ef\u80fd\u88ab\u4f7f\u7528\u7684\u6267\u884c\u8ba1\u5212\u4e2d\u4ee3\u4ef7\u6700\u5c0f\u7684\u6267\u884c\u8ba1\u5212\uff0c\u4ece\u800c\u63d0\u5347\u6027\u80fd\u3002<\/p>\n<p>\u76ee\u524dCBO\u4e3b\u8981\u7684\u4f18\u5316\u70b9\u662fJoin\u7b97\u6cd5\u9009\u62e9\u3002\u4e3e\u4e2a\u7b80\u5355\u4f8b\u5b50\uff0c\u5f53\u4e24\u4e2a\u8868\u505aJoin\u64cd\u4f5c\uff0c\u5982\u679c\u5176\u4e2d\u4e00\u5f20\u539f\u672c\u5f88\u5927\u7684\u8868\u7ecf\u8fc7Filter\u64cd\u4f5c\u4e4b\u540e\u7ed3\u679c\u96c6\u5c0f\u4e8eBroadCast\u7684\u9608\u503c\uff0c\u5728\u6ca1\u6709CBO\u60c5\u51b5\u4e0b\u662f\u65e0\u6cd5\u611f\u77e5\u5927\u8868\u8fc7\u6ee4\u540e\u53d8\u5c0f\u7684\u60c5\u51b5\uff0c\u91c7\u7528\u7684\u662fSortMergeJoin\u7b97\u6cd5\uff0c\u6d89\u53ca\u5230\u5927\u91cfShuffle\u64cd\u4f5c\uff0c\u5f88\u8017\u8d39\u6027\u80fd\uff1b\u5728\u6709CBO\u7684\u60c5\u51b5\u4e0b\u662f\u53ef\u4ee5\u611f\u77e5\u5230\u7ed3\u679c\u96c6\u7684\u53d8\u5316\uff0c\u91c7\u7528\u7684\u662fBroadcastHashJoin\u7b97\u6cd5\uff0c\u4f1a\u5c06\u8fc7\u6ee4\u540e\u7684\u5c0f\u8868BroadCast\u5230\u6bcf\u4e2a\u8282\u70b9\uff0c\u8f6c\u53d8\u4e3a\u975eShuffle\u64cd\u4f5c\uff0c\u4ece\u800c\u5927\u5927\u63d0\u9ad8\u6027\u80fd\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>Spark CBO\u7684\u8bbe\u8ba1\u601d\u8def\u662f\uff0c\u57fa\u4e8e\u8868\u548c\u5217\u7684\u7edf\u8ba1\u4fe1\u606f\uff0c\u5bf9\u5404\u4e2a\u64cd\u4f5c\u7b97\u5b50\uff08Operator\uff09\u4ea7\u751f\u7684\u4e2d\u95f4\u7ed3\u679c\u96c6\u5927\u5c0f\u8fdb\u884c\u4f30\u7b97\uff0c\u6700\u540e\u6839\u636e\u4f30\u7b97\u7684\u7ed3\u679c\u6765\u9009\u62e9\u6700\u4f18\u7684\u6267\u884c\u8ba1\u5212\u3002<\/p>\n<ol>\n<li>\u8bbe\u7f6e\u914d\u7f6e\u9879\u3002<\/li>\n<\/ol>\n<ul>\n<li>\u5728\u201cspark-defaults.conf\u201d\u914d\u7f6e\u6587\u4ef6\u4e2d\u589e\u52a0\u914d\u7f6e\u9879\u201cspark.sql.cbo\u201d\uff0c\u5c06\u5176\u8bbe\u7f6e\u4e3atrue\uff0c\u9ed8\u8ba4\u4e3afalse\u3002<!-- -->\n<ul>\n<li>\u5728\u5ba2\u6237\u7aef\u6267\u884cSQL\u8bed\u53e5<em><strong>set spark.sql.cbo=true<\/strong><\/em>\u8fdb\u884c\u914d\u7f6e\u3002<\/li>\n<\/ul>\n<\/li>\n<li>\u6267\u884c\u7edf\u8ba1\u4fe1\u606f\u751f\u6210\u547d\u4ee4\uff0c\u5f97\u5230\u7edf\u8ba1\u4fe1\u606f\u3002\u00a0\u8bf4\u660e\uff1a<\/li>\n<\/ul>\n<p>\u6b64\u6b65\u9aa4\u53ea\u9700\u5728\u8fd0\u884c\u6240\u6709SQL\u524d\u6267\u884c\u4e00\u6b21\u3002\u5982\u679c\u6570\u636e\u96c6\u53d1\u751f\u4e86\u53d8\u5316\uff08\u63d2\u5165\u3001\u66f4\u65b0\u6216\u5220\u9664\uff09\uff0c\u4e3a\u4fdd\u8bc1CBO\u7684\u4f18\u5316\u6548\u679c\uff0c\u9700\u8981\u5bf9\u6709\u53d8\u5316\u7684\u8868\u6216\u8005\u5217\u518d\u6b21\u6267\u884c\u7edf\u8ba1\u4fe1\u606f\u751f\u6210\u547d\u4ee4\u91cd\u65b0\u751f\u6210\u7edf\u8ba1\u4fe1\u606f\uff0c\u4ee5\u5f97\u5230\u6700\u65b0\u7684\u6570\u636e\u5206\u5e03\u60c5\u51b5\u3002<\/p>\n<ul>\n<li>\n<p>\u8868\uff1a\u6267\u884c<em><strong>COMPUTE STATS FOR TABLE src<\/strong><\/em>\u547d\u4ee4\u8ba1\u7b97\u8868\u7684\u7edf\u8ba1\u4fe1\u606f\uff0c\u7edf\u8ba1\u4fe1\u606f\u5305\u62ec\u8bb0\u5f55\u6761\u6570\u3001\u6587\u4ef6\u6570\u548c\u7269\u7406\u5b58\u50a8\u603b\u5927\u5c0f\u3002<\/p>\n<ul>\n<li>\u5217\uff1a<\/li>\n<\/ul>\n<\/li>\n<li>\n<p>\u6267\u884c<em><strong>COMPUTE STATS FOR TABLE src ON COLUMNS<\/strong><\/em>\u547d\u4ee4\u8ba1\u7b97\u6240\u6709\u5217\u7684\u7edf\u8ba1\u4fe1\u606f\u3002<br \/>\n* \u6267\u884c<em><strong>COMPUTE STATS FOR TABLE src ON COLUMNS<\/strong><\/em>\u00a0<em>name,age<\/em>\u547d\u4ee4\u8ba1\u7b97\u8868\u4e2dname\u548cage\u4e24\u4e2a\u5b57\u6bb5\u7684\u7edf\u8ba1\u4fe1\u606f\u3002<\/p>\n<\/li>\n<\/ul>\n<p>\u5f53\u524d\u5217\u7684\u7edf\u8ba1\u4fe1\u606f\u652f\u6301\u56db\u79cd\u7c7b\u578b\uff1a\u6570\u503c\u7c7b\u578b\u3001\u65e5\u671f\u7c7b\u578b\u3001\u65f6\u95f4\u7c7b\u578b\u548c\u5b57\u7b26\u4e32\u7c7b\u578b\u3002\u5bf9\u4e8e\u6570\u503c\u7c7b\u578b\u3001\u65e5\u671f\u7c7b\u578b\u548c\u65f6\u95f4\u7c7b\u578b\uff0c\u7edf\u8ba1\u4fe1\u606f\u5305\u62ec\uff1aMax\u3001Min\u3001\u4e0d\u540c\u503c\u4e2a\u6570(Number of Distinct Value\uff0cNDV)\u3001\u7a7a\u503c\u4e2a\u6570(Number of Null)\u548cHistogram\uff08\u652f\u6301\u7b49\u5bbd\u3001\u7b49\u9ad8\u76f4\u65b9\u56fe\uff09\uff1b\u5bf9\u4e8e\u5b57\u7b26\u4e32\u7c7b\u578b\uff0c\u7edf\u8ba1\u4fe1\u606f\u5305\u62ec\uff1aMax\u3001Min\u3001Max Length\u3001Average Length\u3001\u4e0d\u540c\u503c\u4e2a\u6570(Number of Distinct Value\uff0cNDV)\u3001\u7a7a\u503c\u4e2a\u6570(Number of Null)\u548cHistogram\uff08\u652f\u6301\u7b49\u5bbd\u76f4\u65b9\u56fe\uff09\u3002<\/p>\n<ul>\n<li>\n<p>CBO\u8c03\u4f18<\/p>\n<\/li>\n<li>\n<p>\u81ea\u52a8\u4f18\u5316\uff1a\u7528\u6237\u6839\u636e\u81ea\u5df1\u7684\u4e1a\u52a1\u573a\u666f\uff0c\u8f93\u5165SQL\u8bed\u53e5\u67e5\u8be2\uff0c\u7a0b\u5e8f\u4f1a\u81ea\u52a8\u53bb\u5224\u65ad\u8f93\u5165\u7684SQL\u8bed\u53e5\u662f\u5426\u7b26\u5408\u4f18\u5316\u7684\u573a\u666f\uff0c\u4ece\u800c\u81ea\u52a8\u9009\u62e9Join\u4f18\u5316\u7b97\u6cd5\u3002<\/p>\n<ul>\n<li>\u624b\u52a8\u4f18\u5316\uff1a\u7528\u6237\u53ef\u4ee5\u901a\u8fc7<em><strong>DESC FORMATTED src<\/strong><\/em>\u547d\u4ee4\u67e5\u770b\u7edf\u8ba1\u4fe1\u606f\uff0c\u6839\u636e\u7edf\u8ba1\u4fe1\u606f\u7684\u5206\u5e03\uff0c\u4eba\u5de5\u4f18\u5316SQL\u8bed\u53e5\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<h3>12.9.5\u00a0Carbon\u6027\u80fd\u8c03\u4f18<\/h3>\n<p>\u67e5\u8be2\u6027\u80fd\u8c03\u4f18<\/p>\n<p>Carbon\u53ef\u4ee5\u901a\u8fc7\u8c03\u6574\u5404\u79cd\u53c2\u6570\u6765\u63d0\u9ad8\u67e5\u8be2\u6027\u80fd\u3002\u5927\u90e8\u5206\u53c2\u6570\u805a\u7126\u4e8e\u589e\u52a0\u5e76\u884c\u6027\u5904\u7406\u548c\u66f4\u597d\u5730\u4f7f\u7528\u7cfb\u7edf\u8d44\u6e90\u3002<\/p>\n<ul>\n<li>\u5b57\u5178\uff08Dictionary\uff09\u7f13\u5b58\uff1aCarbon\u91c7\u7528\u5b57\u5178\u7f16\u7801\uff0c\u4ee5\u63d0\u5347\u67e5\u8be2\u6027\u80fd\u548c\u6570\u636e\u6587\u4ef6\u538b\u7f29\u7387\u3002\u5728\u4e00\u4e2a\u8868\u683c\u4e2d\uff0cCarbon\u4f1a\u5bf9\u6bcf\u4e2a\u5b57\u5178\u7f16\u7801\u5217\u521b\u5efa\u5b57\u5178\u6587\u4ef6\u3002\u8fd9\u4e9b\u5b57\u5178\u6587\u4ef6\u901a\u8fc7\u89e3\u7801\u67e5\u8be2\u6267\u884c\u7ed3\u679c\uff0c\u7531\u67e5\u8be2\u5904\u7406\u5668\u52a0\u8f7d\u5230\u5185\u5b58\u4e2d\u3002\u4e00\u65e6\u5b8c\u6210\u52a0\u8f7d\uff0c\u8fd9\u4e9b\u5b57\u5178\u6587\u4ef6\u4f1a\u88ab\u4fdd\u5b58\u5728\u5185\u5b58\u4e2d\uff0c\u907f\u514d\u88ab\u518d\u6b21\u4ece\u78c1\u76d8\u8bfb\u53d6\uff0c\u4ee5\u52a0\u901f\u67e5\u8be2\u6267\u884c\u7684\u901f\u5ea6\u3002\u4f46\u662f\uff0c\u5982\u679c\u8868\u7684\u6570\u91cf\u8f83\u591a\uff0c\u5219\u4e0d\u80fd\u5c06\u6240\u6709\u5217\u5b57\u5178\u6587\u4ef6\u90fd\u4fdd\u5b58\u81f3\u7269\u7406\u5185\u5b58\u4e2d\u3002\u56e0\u6b64\uff0cCarbon\u5bf9\u4fdd\u5b58\u5728\u5176\u5185\u5b58\u7f13\u5b58\u4e2d\u7684\u5217\u5b57\u5178\u6587\u4ef6\u7684\u6570\u76ee\u6709\u9650\u5236\u3002\u8be5\u9650\u5236\u503c\u53ef\u901a\u8fc7\u4e0b\u5217\u5c5e\u6027\u53c2\u6570\u8fdb\u884c\u914d\u7f6e\u3002\u8bbe\u7f6e\u4e00\u4e2a\u8f83\u5927\u7684\u503c\uff0c\u53ef\u5728\u5185\u5b58\u4e2d\u7f13\u5b58\u66f4\u591a\u5217\u5b57\u5178\u6587\u4ef6\u6570\u76ee\uff0c\u8fdb\u800c\u63d0\u5347\u67e5\u8be2\u6027\u80fd\u3002\u5355\u4f4d\u4e3aMB\uff0c\u9ed8\u8ba4\u503c\u4e3a0\u3002\u4f8b\u5982\uff0c\u53ef\u914d\u7f6e\u4e3a\u201ccarbon.max.level.cache.size=[10]\u201d\u3002<\/li>\n<li>Spark Executor\u6570\u91cf\uff1aExecutor\u662fSpark\u5e76\u884c\u6027\u7684\u57fa\u7840\u5b9e\u4f53\u3002\u901a\u8fc7\u589e\u52a0Executor\u6570\u91cf\uff0c\u96c6\u7fa4\u4e2d\u7684\u5e76\u884c\u6570\u91cf\u4e5f\u4f1a\u589e\u52a0\u3002\u5173\u4e8e\u5982\u4f55\u914d\u7f6eExecutor\u6570\u91cf\uff0c\u8bf7\u53c2\u8003Spark\u8d44\u6599\u3002<\/li>\n<li>Executor\u6838\uff1a\u6bcf\u4e2aExecutor\u5185\uff0c\u5e76\u884c\u4efb\u52a1\u6570\u53d7Executor\u6838\u7684\u914d\u7f6e\u63a7\u5236\u3002\u901a\u8fc7\u589e\u52a0Executor\u6838\u6570\uff0c\u53ef\u589e\u52a0\u5e76\u884c\u4efb\u52a1\u6570\uff0c\u4ece\u800c\u63d0\u9ad8\u6027\u80fd\u3002\u5173\u4e8e\u5982\u4f55\u914d\u7f6eExecutor\u6838\u6570\uff0c\u8bf7\u53c2\u8003Spark\u8d44\u6599\u3002<\/li>\n<li>HDFS block\u5bb9\u91cf\uff1aCarbon\u901a\u8fc7\u7ed9\u4e0d\u540c\u7684\u5904\u7406\u5668\u5206\u914d\u4e0d\u540c\u7684block\u6765\u5206\u914d\u67e5\u8be2\u4efb\u52a1\u3002\u6240\u4ee5\u4e00\u4e2aHDFS block\u662f\u4e00\u4e2a\u5206\u533a\u5355\u5143\u3002\u53e6\u5916\uff0cCarbon\u5728Spark\u9a71\u52a8\u5668\u4e2d\uff0c\u652f\u6301\u5168\u5c40block\u7ea7\u7d22\u5f15\uff0c\u8fd9\u6709\u52a9\u4e8e\u51cf\u5c11\u9700\u8981\u88ab\u626b\u63cf\u7684\u67e5\u8be2block\u7684\u6570\u91cf\u3002\u8bbe\u7f6e\u8f83\u5927\u7684block\u5bb9\u91cf\uff0c\u53ef\u63d0\u9ad8I\/O\u6548\u7387\uff0c\u4f46\u662f\u4f1a\u964d\u4f4e\u5168\u5c40\u7d22\u5f15\u6548\u7387\uff1b\u8bbe\u7f6e\u8f83\u5c0f\u7684block\u5bb9\u91cf\uff0c\u610f\u5473\u7740\u66f4\u591a\u7684block\u6570\u91cf\uff0c\u4f1a\u964d\u4f4eI\/O\u6548\u7387\uff0c\u4f46\u662f\u4f1a\u63d0\u9ad8\u5168\u5c40\u7d22\u5f15\u6548\u7387\uff0c\u540c\u65f6\uff0c\u5bf9\u4e8e\u7d22\u5f15\u67e5\u8be2\u4f1a\u8981\u6c42\u66f4\u591a\u7684\u5185\u5b58\u3002<\/li>\n<li>\u626b\u63cf\u7ebf\u7a0b\u6570\u91cf\uff1a\u626b\u63cf\u4eea\uff08Scanner\uff09\u7ebf\u7a0b\u63a7\u5236\u6bcf\u4e2a\u4efb\u52a1\u4e2d\u5e76\u884c\u5904\u7406\u7684\u6570\u636e\u5757\u7684\u6570\u91cf\u3002\u901a\u8fc7\u589e\u52a0\u626b\u63cf\u4eea\u7ebf\u7a0b\u6570\uff0c\u53ef\u589e\u52a0\u5e76\u884c\u5904\u7406\u7684\u6570\u636e\u5757\u7684\u6570\u91cf\uff0c\u4ece\u800c\u63d0\u9ad8\u6027\u80fd\u3002\u53ef\u4f7f\u7528\u201ccarbon.properties\u201d\u6587\u4ef6\u4e2d\u7684\u201ccarbon.number.of.cores\u201d\u5c5e\u6027\u6765\u914d\u7f6e\u626b\u63cf\u4eea\u7ebf\u7a0b\u6570\u3002\u4f8b\u5982\uff0c\u201ccarbon.number.of.cores = 4\u201d\u3002<\/li>\n<\/ul>\n<p>Carbon\u67e5\u8be2\u6d41\u7a0b<\/p>\n<p>\u5f53Carbon\u9996\u6b21\u6536\u5230\u5bf9\u67d0\u4e2a\u8868\uff08\u4f8b\u5982\u8868A\uff09\u7684\u67e5\u8be2\u4efb\u52a1\u65f6\uff0c\u7cfb\u7edf\u4f1a\u52a0\u8f7d\u8868A\u7684\u7d22\u5f15\u6570\u636e\u5230\u5185\u5b58\u4e2d\uff0c\u6267\u884c\u67e5\u8be2\u6d41\u7a0b\u3002\u5f53Carbon\u518d\u6b21\u6536\u5230\u5bf9\u8868A\u7684\u67e5\u8be2\u4efb\u52a1\u65f6\uff0c\u7cfb\u7edf\u5219\u4e0d\u9700\u8981\u518d\u52a0\u8f7d\u5176\u7d22\u5f15\u6570\u636e\u3002<\/p>\n<p>\u5728Carbon\u4e2d\u6267\u884c\u67e5\u8be2\u65f6\uff0c\u67e5\u8be2\u4efb\u52a1\u4f1a\u88ab\u5206\u6210\u51e0\u4e2a\u626b\u63cf\u4efb\u52a1\u3002\u5373\uff0c\u57fa\u4e8eCarbon\u6570\u636e\u5b58\u50a8\u7684HDFS block\u5bf9\u626b\u63cf\u4efb\u52a1\u8fdb\u884c\u5206\u5272\u3002\u626b\u63cf\u4efb\u52a1\u7531\u96c6\u7fa4\u4e2d\u7684\u6267\u884c\u5668\u6267\u884c\u3002\u626b\u63cf\u4efb\u52a1\u53ef\u4ee5\u5e76\u884c\u3001\u90e8\u5206\u5e76\u884c\uff0c\u6216\u987a\u5e8f\u5904\u7406\uff0c\u5177\u4f53\u91c7\u7528\u7684\u65b9\u5f0f\u53d6\u51b3\u4e8e\u6267\u884c\u5668\u7684\u6570\u91cf\u4ee5\u53ca\u914d\u7f6e\u7684\u6267\u884c\u5668\u6838\u6570\u3002<\/p>\n<p>\u67e5\u8be2\u4efb\u52a1\u7684\u67d0\u4e9b\u90e8\u5206\u53ef\u5728\u72ec\u7acb\u7684\u4efb\u52a1\u7ea7\u4e0a\u5904\u7406\uff0c\u4f8b\u5982select\u548cfilter\u3002\u67e5\u8be2\u4efb\u52a1\u7684\u67d0\u4e9b\u90e8\u5206\u53ef\u5728\u72ec\u7acb\u7684\u4efb\u52a1\u7ea7\u4e0a\u8fdb\u884c\u90e8\u5206\u5904\u7406\uff0c\u4f8b\u5982group-by\u3001count\u3001distinct count\u7b49\u3002<\/p>\n<p>\u67d0\u4e9b\u64cd\u4f5c\u65e0\u6cd5\u5728\u4efb\u52a1\u7ea7\u4e0a\u5904\u7406\uff0c\u4f8b\u5982Having Clause\uff08\u5206\u7ec4\u540e\u7684\u8fc7\u6ee4\uff09\uff0csort\u7b49\u3002\u8fd9\u4e9b\u65e0\u6cd5\u5728\u4efb\u52a1\u7ea7\u4e0a\u5904\u7406\uff0c\u6216\u53ea\u80fd\u5728\u4efb\u52a1\u7ea7\u4e0a\u90e8\u5206\u5904\u7406\u7684\u64cd\u4f5c\u9700\u8981\u5728\u96c6\u7fa4\u5185\u8de8\u6267\u884c\u5668\u6765\u4f20\u8f93\u6570\u636e\uff08\u90e8\u5206\u7ed3\u679c\uff09\u3002\u8fd9\u4e2a\u4f20\u9001\u64cd\u4f5c\u88ab\u79f0\u4e3ashuffle\u3002<\/p>\n<p>\u4efb\u52a1\u6570\u91cf\u8d8a\u591a\uff0c\u9700\u8981shuffle\u7684\u6570\u636e\u5c31\u8d8a\u591a\uff0c\u4f1a\u5bf9\u67e5\u8be2\u6027\u80fd\u4ea7\u751f\u4e0d\u5229\u5f71\u54cd\u3002<\/p>\n<p>\u7531\u4e8e\u4efb\u52a1\u6570\u91cf\u53d6\u51b3\u4e8eHDFS block\u7684\u6570\u91cf\uff0c\u800cHDFS block\u7684\u6570\u91cf\u53d6\u51b3\u4e8e\u6bcf\u4e2ablock\u7684\u5927\u5c0f\uff0c\u56e0\u6b64\u5408\u7406\u9009\u62e9HDFS block\u7684\u5927\u5c0f\u5f88\u91cd\u8981\uff0c\u9700\u8981\u5728\u63d0\u9ad8\u5e76\u884c\u6027\uff0c\u8fdb\u884cshuffle\u64cd\u4f5c\u7684\u6570\u636e\u91cf\u548c\u805a\u5408\u8868\u7684\u5927\u5c0f\u4e4b\u95f4\u8fbe\u5230\u5e73\u8861\u3002<\/p>\n<p>\u5206\u5272\u548cExecutors\u7684\u5173\u7cfb<\/p>\n<p>\u5982\u679c\u5206\u5272\u6570\u2264Executor\u6570xExecutor\u6838\u6570\uff0c\u90a3\u4e48\u4efb\u52a1\u5c06\u4ee5\u5e76\u884c\u65b9\u5f0f\u8fd0\u884c\u3002\u5426\u5219\uff0c\u67d0\u4e9b\u4efb\u52a1\u53ea\u6709\u5728\u5176\u4ed6\u4efb\u52a1\u5b8c\u6210\u4e4b\u540e\u624d\u80fd\u5f00\u59cb\u3002\u56e0\u6b64\uff0c\u8981\u786e\u4fddExecutor\u6570xExecutor\u6838\u6570\u2265\u5206\u5272\u6570\u3002\u540c\u65f6\uff0c\u8fd8\u8981\u786e\u4fdd\u6709\u8db3\u591f\u7684\u5206\u5272\u6570\uff0c\u8fd9\u6837\u4e00\u4e2a\u67e5\u8be2\u4efb\u52a1\u53ef\u88ab\u5206\u4e3a\u8db3\u591f\u591a\u7684\u5b50\u4efb\u52a1\uff0c\u4ece\u800c\u786e\u4fdd\u5e76\u884c\u6027\u3002<\/p>\n<p>\u914d\u7f6e\u626b\u63cf\u4eea\u7ebf\u7a0b<\/p>\n<p>\u626b\u63cf\u4eea\u7ebf\u7a0b\u5c5e\u6027\u51b3\u5b9a\u4e86\u6bcf\u4e2a\u5206\u5272\u7684\u6570\u636e\u88ab\u5212\u5206\u7684\u53ef\u5e76\u884c\u5904\u7406\u7684\u6570\u636e\u5757\u7684\u6570\u91cf\u3002\u5982\u679c\u6570\u91cf\u8fc7\u591a\uff0c\u4f1a\u4ea7\u751f\u5f88\u591a\u5c0f\u6570\u636e\u5757\uff0c\u6027\u80fd\u4f1a\u53d7\u5230\u5f71\u54cd\u3002\u5982\u679c\u6570\u91cf\u8fc7\u5c11\uff0c\u5e76\u884c\u6027\u4e0d\u4f73\uff0c\u6027\u80fd\u4e5f\u4f1a\u53d7\u5230\u5f71\u54cd\u3002\u56e0\u6b64\uff0c\u51b3\u5b9a\u626b\u63cf\u4eea\u7ebf\u7a0b\u6570\u65f6\uff0c\u6700\u597d\u8003\u8651\u4e00\u4e2a\u5206\u5272\u5185\u7684\u5e73\u5747\u6570\u636e\u5927\u5c0f\uff0c\u9009\u62e9\u4e00\u4e2a\u4f7f\u6570\u636e\u5757\u4e0d\u4f1a\u5f88\u5c0f\u7684\u503c\u3002\u7ecf\u9a8c\u6cd5\u5219\u662f\u5c06\u5355\u4e2a\u5757\u5927\u5c0f\uff08MB\uff09\u9664\u4ee5250\u5f97\u5230\u7684\u503c\u4f5c\u4e3a\u626b\u63cf\u4eea\u7ebf\u7a0b\u6570\u3002<\/p>\n<p>\u589e\u52a0\u5e76\u884c\u6027\u8fd8\u9700\u8003\u8651\u7684\u91cd\u8981\u4e00\u70b9\u662f\u96c6\u7fa4\u4e2d\u5b9e\u9645\u53ef\u7528\u7684CPU\u6838\u6570\uff0c\u786e\u4fdd\u5e76\u884c\u8ba1\u7b97\u6570\u4e0d\u8d85\u8fc7\u5b9e\u9645CPU\u6838\u6570\u768475%\u81f380%\u3002<\/p>\n<p>CPU\u6838\u6570\u7ea6\u7b49\u4e8e\uff1a<\/p>\n<p>\u5e76\u884c\u4efb\u52a1\u6570x\u626b\u63cf\u4eea\u7ebf\u7a0b\u6570\u3002\u5176\u4e2d\u5e76\u884c\u4efb\u52a1\u6570\u4e3a\u5206\u5272\u6570\u548c\u6267\u884c\u5668\u6570x\u6267\u884c\u5668\u6838\u6570\u4e24\u8005\u4e4b\u95f4\u7684\u8f83\u5c0f\u503c\u3002<\/p>\n<p>\u6570\u636e\u52a0\u8f7d\u6027\u80fd\u8c03\u4f18<\/p>\n<p>\u6570\u636e\u52a0\u8f7d\u6027\u80fd\u8c03\u4f18\u4e0e\u67e5\u8be2\u6027\u80fd\u8c03\u4f18\u5dee\u5f02\u5f88\u5927\u3002\u8ddf\u67e5\u8be2\u6027\u80fd\u4e00\u6837\uff0c\u6570\u636e\u52a0\u8f7d\u6027\u80fd\u4e5f\u53d6\u51b3\u4e8e\u53ef\u8fbe\u5230\u7684\u5e76\u884c\u6027\u3002\u5728\u6570\u636e\u52a0\u8f7d\u60c5\u51b5\u4e0b\uff0c\u5de5\u4f5c\u7ebf\u7a0b\u7684\u6570\u91cf\u51b3\u5b9a\u5e76\u884c\u7684\u5355\u5143\u3002\u56e0\u6b64\uff0c\u66f4\u591a\u7684\u6267\u884c\u5668\u5c31\u610f\u5473\u7740\u66f4\u591a\u7684\u6267\u884c\u5668\u6838\u6570\uff0c\u6bcf\u4e2a\u6267\u884c\u5668\u90fd\u53ef\u4ee5\u63d0\u9ad8\u6570\u636e\u52a0\u8f7d\u6027\u80fd\u3002<\/p>\n<p>\u540c\u65f6\uff0c\u4e3a\u4e86\u5f97\u5230\u66f4\u597d\u7684\u6027\u80fd\uff0c\u53ef\u5728HDFS\u4e2d\u914d\u7f6e\u5982\u4e0b\u53c2\u6570\u3002<\/p>\n<p>\u886812-39\u00a0HDFS\u914d\u7f6e<\/p>\n<p>dfs.datanode.drop.cache.behind.reads<br \/>\nfalse<br \/>\ndfs.datanode.drop.cache.behind.writes<br \/>\nfalse<br \/>\ndfs.datanode.sync.behind.writes<br \/>\nfalse<\/p>\n<p>\u538b\u7f29\u8c03\u4f18<\/p>\n<p>Carbon\u7ed3\u5408\u5c11\u6570\u8f7b\u91cf\u7ea7\u538b\u7f29\u7b97\u6cd5\u548c\u91cd\u91cf\u7ea7\u538b\u7f29\u7b97\u6cd5\u6765\u538b\u7f29\u6570\u636e\u3002\u867d\u7136\u8fd9\u4e9b\u7b97\u6cd5\u53ef\u5904\u7406\u4efb\u4f55\u7c7b\u578b\u7684\u6570\u636e\uff0c\u4f46\u5982\u679c\u6570\u636e\u7ecf\u8fc7\u6392\u5e8f\uff0c\u76f8\u4f3c\u503c\u5728\u4e00\u8d77\u51fa\u73b0\u65f6\uff0c\u5c31\u4f1a\u83b7\u5f97\u66f4\u597d\u7684\u538b\u7f29\u7387\u3002<\/p>\n<p>Carbon\u6570\u636e\u52a0\u8f7d\u8fc7\u7a0b\u4e2d\uff0c\u6570\u636e\u57fa\u4e8eTable\u4e2d\u7684\u5217\u987a\u5e8f\u8fdb\u884c\u6392\u5e8f\uff0c\u4ece\u800c\u786e\u4fdd\u76f8\u4f3c\u503c\u5728\u4e00\u8d77\u51fa\u73b0\uff0c\u4ee5\u83b7\u5f97\u66f4\u597d\u7684\u538b\u7f29\u7387\u3002<\/p>\n<p>\u7531\u4e8eCarbon\u6309\u7167Table\u4e2d\u5b9a\u4e49\u7684\u5217\u987a\u5e8f\u5c06\u6570\u636e\u8fdb\u884c\u6392\u5e8f\uff0c\u56e0\u6b64\u5217\u987a\u5e8f\u5bf9\u4e8e\u538b\u7f29\u6548\u7387\u8d77\u91cd\u8981\u4f5c\u7528\u3002\u5982\u679c\u4f4e\u57fa\u6570\u7ef4\u5ea6\u4f4d\u4e8e\u5de6\u8fb9\uff0c\u90a3\u4e48\u6392\u5e8f\u540e\u7684\u6570\u636e\u5206\u533a\u8303\u56f4\u8f83\u5c0f\uff0c\u538b\u7f29\u6548\u7387\u8f83\u9ad8\u3002\u5982\u679c\u9ad8\u57fa\u6570\u7ef4\u5ea6\u4f4d\u4e8e\u5de6\u8fb9\uff0c\u90a3\u4e48\u6392\u5e8f\u540e\u7684\u6570\u636e\u5206\u533a\u8303\u56f4\u8f83\u5927\uff0c\u538b\u7f29\u6548\u7387\u8f83\u4f4e\u3002<\/p>\n<p>\u5185\u5b58\u8c03\u4f18<\/p>\n<p>Carbon\u4e3a\u5185\u5b58\u8c03\u4f18\u63d0\u4f9b\u4e86\u4e00\u4e2a\u673a\u5236\uff0c\u5176\u4e2d\u6570\u636e\u52a0\u8f7d\u4f1a\u4f9d\u8d56\u4e8e\u67e5\u8be2\u4e2d\u9700\u8981\u7684\u5217\u3002\u4e0d\u8bba\u4f55\u65f6\uff0c\u63a5\u6536\u5230\u4e00\u4e2a\u67e5\u8be2\u547d\u4ee4\uff0c\u5c06\u4f1a\u83b7\u53d6\u5230\u8be5\u67e5\u8be2\u4e2d\u7684\u5217\uff0c\u5e76\u786e\u4fdd\u5185\u5b58\u4e2d\u8fd9\u4e9b\u5217\u6709\u6570\u636e\u52a0\u8f7d\u3002\u5728\u8be5\u64cd\u4f5c\u671f\u95f4\uff0c\u5982\u679c\u8fbe\u5230\u5185\u5b58\u7684\u9608\u503c\uff0c\u4e3a\u4e86\u7ed9\u67e5\u8be2\u9700\u8981\u7684\u5217\u63d0\u4f9b\u5185\u5b58\u7a7a\u95f4\uff0c\u6700\u5c11\u4f7f\u7528\u52a0\u8f7d\u7ea7\u522b\u7684\u6587\u4ef6\u5c06\u4f1a\u88ab\u5220\u9664\u3002<\/p>\n<h2>12.10\u00a0Storm<\/h2>\n<h3>12.10.1\u00a0Storm\u6027\u80fd\u8c03\u4f18<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u901a\u8fc7\u8c03\u6574Storm\u53c2\u6570\u8bbe\u7f6e\uff0c\u53ef\u4ee5\u63d0\u5347\u7279\u5b9a\u4e1a\u52a1\u573a\u666f\u4e0bStorm\u7684\u6027\u80fd\u3002<\/p>\n<p>Storm\u53c2\u6570\u5165\u53e3\uff1a\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; Storm &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002<\/p>\n<p>\u62d3\u6251\u8c03\u4f18<\/p>\n<p>\u5f53\u9700\u8981\u63d0\u5347Storm\u6570\u636e\u91cf\u5904\u7406\u6027\u80fd\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7\u62d3\u6251\u8c03\u4f18\u7684\u64cd\u4f5c\u63d0\u9ad8\u6548\u7387\u3002\u5efa\u8bae\u5728\u53ef\u9760\u6027\u8981\u6c42\u4e0d\u9ad8\u7684\u573a\u666f\u4e0b\u8fdb\u884c\u4f18\u5316\u3002<\/p>\n<p>\u886812-40\u00a0\u8c03\u4f18\u53c2\u6570<\/p>\n<p>topology.acker.executors<br \/>\nnull<br \/>\nAcker\u7684\u6267\u884c\u5668\u6570\u91cf\u3002\u5f53\u4e1a\u52a1\u5e94\u7528\u5bf9\u53ef\u9760\u6027\u8981\u6c42\u8f83\u4f4e\uff0c\u5141\u8bb8\u4e0d\u5904\u7406\u90e8\u5206\u6570\u636e\uff0c\u53ef\u8bbe\u7f6e\u53c2\u6570\u503c\u4e3a\u201cnull\u201d\u6216\u201c0\u201d\uff0c\u4ee5\u5173\u95edAcker\u7684\u6267\u884c\u5668\uff0c\u51cf\u5c11\u6d41\u63a7\u5236\uff0c\u4e0d\u7edf\u8ba1\u6d88\u606f\u65f6\u5ef6\uff0c\u63d0\u9ad8\u6027\u80fd\u3002<br \/>\ntopology.max.spout.pending<br \/>\nnull<br \/>\nSpout\u6d88\u606f\u7f13\u5b58\u6570\uff0c\u4ec5\u5728Acker\u4e0d\u4e3a0\u6216\u8005\u4e0d\u4e3anull\u7684\u60c5\u51b5\u4e0b\u751f\u6548\u3002Spout\u5c06\u53d1\u9001\u5230\u4e0b\u6e38Bolt\u7684\u6bcf\u6761\u6d88\u606f\u52a0\u5165\u5230pending\u961f\u5217\uff0c\u5f85\u4e0b\u6e38Bolt\u5904\u7406\u5b8c\u6210\u5e76\u786e\u8ba4\u540e\uff0c\u518d\u4ecepending\u961f\u5217\u79fb\u9664\uff0c\u5f53pending\u961f\u5217\u5360\u6ee1\u65f6Spout\u6682\u505c\u6d88\u606f\u53d1\u9001\u3002\u589e\u52a0pending\u503c\u53ef\u63d0\u9ad8Spout\u7684\u6bcf\u79d2\u6d88\u606f\u541e\u5410\u91cf\uff0c\u63d0\u9ad8\u6027\u80fd\uff0c\u4f46\u5ef6\u65f6\u540c\u6b65\u589e\u52a0\u3002<br \/>\ntopology.transfer.buffer.size<br \/>\n32<br \/>\n\u6bcf\u4e2aworker\u8fdb\u7a0bDisruptor\u6d88\u606f\u961f\u5217\u5927\u5c0f\uff0c\u5efa\u8bae\u57284\u523032\u4e4b\u95f4\uff0c\u589e\u5927\u6d88\u606f\u961f\u5217\u53ef\u4ee5\u63d0\u5347\u541e\u5410\u91cf\uff0c\u4f46\u5ef6\u65f6\u53ef\u80fd\u4f1a\u589e\u52a0\u3002<br \/>\nRES_CPUSET_PERCENTAGE<br \/>\n80<br \/>\n\u8bbe\u7f6e\u5404\u4e2a\u8282\u70b9\u4e0a\u7684Supervisor\u89d2\u8272\u5b9e\u4f8b\uff08\u5305\u542b\u5176\u542f\u52a8\u5e76\u7ba1\u7406\u7684Worker\u8fdb\u7a0b\uff09\u6240\u4f7f\u7528\u7684\u7269\u7406CPU\u767e\u5206\u6bd4\u3002\u6839\u636eSupervisor\u6240\u5728\u8282\u70b9\u4e1a\u52a1\u91cf\u9700\u6c42\uff0c\u9002\u5f53\u8c03\u6574\u53c2\u6570\u503c\uff0c\u4f18\u5316CPU\u4f7f\u7528\u7387\u3002<\/p>\n<p>JVM\u8c03\u4f18<\/p>\n<p>\u5f53\u5e94\u7528\u7a0b\u5e8f\u9700\u8981\u5904\u7406\u5927\u91cf\u6570\u636e\u4ece\u800c\u5360\u7528\u66f4\u591a\u7684\u5185\u5b58\u65f6\uff0c\u5b58\u5728worker\u5185\u5b58\u5927\u4e8e2GB\u7684\u60c5\u51b5\uff0c\u63a8\u8350\u4f7f\u7528G1\u5783\u573e\u56de\u6536\u7b97\u6cd5\u3002<\/p>\n<p>\u886812-41\u00a0\u8c03\u4f18\u53c2\u6570<\/p>\n<p>WORKER_GC_OPTS<br \/>\n-Xms1G -Xmx1G -XX:+UseG1GC -XX:+PrintGCDetails -Xloggc:artifacts\/gc.log -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=artifacts\/heapdump<br \/>\n\u5e94\u7528\u7a0b\u5e8f\u5185\u5b58\u4e2d\u9700\u8981\u4fdd\u5b58\u5927\u91cf\u6570\u636e\uff0cworker\u8fdb\u7a0b\u4f7f\u7528\u7684\u5185\u5b58\u5927\u4e8e2G\uff0c\u90a3\u4e48\u5efa\u8bae\u4f7f\u7528G1\u5783\u573e\u56de\u6536\u7b97\u6cd5\uff0c\u53ef\u4fee\u6539\u53c2\u6570\u503c\u4e3a\u201c-Xms2G -Xmx5G -XX:+UseG1GC\u201d\u3002<\/p>\n<h2>12.11\u00a0YARN<\/h2>\n<h3>12.11.1\u00a0\u901a\u8fc7\u5bb9\u5668\u53ef\u91cd\u7528\u6027\u63d0\u9ad8\u4efb\u52a1\u7684\u5b8c\u6210\u6548\u7387<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u5bb9\u5668\u53ef\u91cd\u7528\u4e0e\u4efb\u52a1\u4f18\u5148\u7ea7\u529f\u80fd\u4e0d\u80fd\u540c\u65f6\u542f\u7528\u3002\u5982\u679c\u540c\u65f6\u542f\u7528\uff0c\u5bb9\u5668\u53ef\u91cd\u7528\u529f\u80fd\u53ef\u6b63\u5e38\u4f7f\u7528\uff0c\u4efb\u52a1\u4f18\u5148\u7ea7\u8bbe\u7f6e\u5c06\u4f1a\u5931\u6548\u3002<\/p>\n<p>\u5bb9\u5668\u53ef\u91cd\u7528\u6027\u53ef\u4ee5\u63d0\u9ad8\u4efb\u52a1\u5b8c\u6210\u7684\u901f\u5ea6\u3002\u5176\u4f18\u52bf\u5982\u4e0b\u6240\u793a\uff1a<\/p>\n<ul>\n<li>\u5bf9\u4e8eHBase\u6570\u636e\u6279\u91cf\u52a0\u8f7d\u800c\u8a00\uff0c\u5bb9\u5668\u53ef\u91cd\u7528\u6027\u7684\u7279\u6027\u53ef\u4f5c\u7528\u4e8e\u4e00\u4e9bMap\u4ee5\u53caReduce\u4efb\u52a1\uff0c\u4f7f\u5176\u80fd\u5feb\u901f\u5b8c\u6210\u4efb\u52a1\u4ece\u800c\u7f29\u77edHBase\u6570\u636e\u6279\u91cf\u52a0\u8f7d\u7684\u65f6\u95f4\u3002<\/li>\n<li>\u51cf\u5c11\u5bb9\u5668\u8c03\u5ea6\u7684\u65f6\u95f4\u548c\u521d\u59cb\u5316\u7684\u65f6\u95f4\u3002<\/li>\n<\/ul>\n<p>\u4e00\u65e6MapReduce\u4f5c\u4e1a\u88ab\u63d0\u4ea4\u3002\u5b83\u5c06\u5206\u53d1\u81f3Map\u548cReduce\u4efb\u52a1\u4e2d\u3002\u7136\u540e\u5e94\u7528\u7ba1\u7406\u5668\uff08\u4ee5\u4e0b\u7b80\u79f0AM\uff09\u5c06\u6267\u884c\u5982\u4e0b\u64cd\u4f5c\u3002<\/p>\n<ol>\n<li>AM\u5411\u8d44\u6e90\u7ba1\u7406\u5668\uff08RM\uff09\u7533\u8bf7\u5bb9\u5668\u53bb\u6267\u884c\u4efb\u52a1\u3002\u6240\u6709\u5bb9\u5668\u5c06\u4e00\u8d77\u5b8c\u6210\u8fd9\u4e2a\u8bf7\u6c42\u3002<\/li>\n<li>RM\u6307\u5b9a\u5bb9\u5668\uff0c\u4e4b\u540eAM\u5c06\u4f1a\u8054\u7cfb\u8282\u70b9\u7ba1\u7406\u5668\uff08NM\uff09\u53bb\u542f\u52a8\u5bb9\u5668\u3002<\/li>\n<li>\u5bb9\u5668\u542f\u52a8\u5b8c\u6bd5\uff0cNM\u4eceAM\u62c9\u53d6\u5e76\u6267\u884c\u4efb\u52a1\u3002<\/li>\n<li>\u4efb\u52a1\u6267\u884c\u5b8c\u6bd5\uff0c\u8fd0\u884c\u8be5\u4efb\u52a1\u7684\u5bb9\u5668\u4e0d\u4f1a\u88ab\u7acb\u5373\u7ec8\u6b62\uff0c\u800c\u662f\u5c1d\u8bd5\u5411AM\u62c9\u53d6\u4e0b\u4e00\u4e2a\u4efb\u52a1\u3002<\/li>\n<\/ol>\n<ul>\n<li>\u82e5\u5bb9\u5668\u83b7\u53d6\u5230\u65b0\u4efb\u52a1\uff0c\u5219\u8be5\u5bb9\u5668\u4f1a\u81ea\u6211\u6e05\u7a7a\u5e76\u521d\u59cb\u5316\uff0c\u4ee5\u9002\u7528\u4e8e\u65b0\u4efb\u52a1\u3002<!-- -->\n<ul>\n<li>\u82e5\u5bb9\u5668\u4e0d\u80fd\u83b7\u53d6\u5230\u65b0\u4efb\u52a1\uff0c\u5219\u4f1a\u8bf7\u6c42\u7ec8\u6b62\u81ea\u5df1\u7684\u8fd0\u884c\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u5f00\u542f\u5bb9\u5668\u53ef\u91cd\u7528\u6027\u914d\u7f6e\u9879\u3002<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; Yarn &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\uff0c\u4fee\u6539\u886812-42\u53c2\u6570\u503c\uff0c\u7136\u540e\u91cd\u65b0\u4e0b\u8f7d\u5e76\u5b89\u88c5Yarn\u5ba2\u6237\u7aef\uff0c\u53c2\u6570\u914d\u7f6e\u751f\u6548\u3002\u6216\u76f4\u63a5\u5728\u5ba2\u6237\u7aef\u76ee\u5f55\u4e0b\u4fee\u6539\uff1a\u5982\u201c\/opt\/client\/Yarn\/config\/mapred-site.xml\u201d\u6587\u4ef6\u91cc\u4fee\u6539\u886812-42\u53c2\u6570\u3002<\/p>\n<p>\u886812-42\u00a0\u5bb9\u5668\u91cd\u7528\u914d\u7f6e<\/p>\n<p>mapreduce.container.reuse.enabled<br \/>\n\u8be5\u914d\u7f6e\u9879\u8bbe\u7f6e\u4e3a\u201ctrue\u201d\u5219\u5bb9\u5668\u53ef\u91cd\u7528\uff0c\u53cd\u4e4b\u5bb9\u5668\u4e0d\u53ef\u91cd\u7528\u3002<br \/>\nfalse<br \/>\nmapreduce.container.reuse.enforce.strict-locality<br \/>\n\u8be5\u914d\u7f6e\u9879\u6307\u5b9a\u662f\u5426\u9075\u5faa\u4e25\u683c\u6570\u636e\u672c\u5730\u5316\u3002\u5982\u679c\u8bbe\u7f6e\u4e3a\u201ctrue\u201d\uff0c\u5219\u53ea\u6709\u672c\u5730\u8282\u70b9\u4e0a\u7684\u4efb\u52a1\u80fd\u88ab\u5206\u914d\u5230\u5bb9\u5668\u3002<br \/>\nfalse<\/p>\n<p>\u4f7f\u7528\u7ea6\u675f<\/p>\n<ul>\n<li>\u5bb9\u5668\u7684\u91cd\u7528\u5c06\u53d7\u9650\u4e8ejob\u4e2d\u4efb\u52a1\u7684\u603b\u6570\u3002\u5bb9\u5668\u4e0d\u4f1a\u88ab\u4e0d\u540c\u7684job\u5171\u4eab\u3002<\/li>\n<li>\u5bb9\u5668\u4e0d\u80fd\u88ab\u5931\u8d25\u7684\u4efb\u52a1\u91cd\u7528\u3002<\/li>\n<li>\u8bbe\u8ba1\u4efb\u52a1\u65f6\uff0c\u5e94\u4fdd\u8bc1\u4e00\u4e2a\u4efb\u52a1\u5b8c\u6210\u540e\u5806\u6808\u91cc\u4e0d\u5b58\u7559\u4efb\u4f55\u5bf9\u8c61\u3002\u8fd9\u5bf9\u4e8e\u5bb9\u5668\u91cd\u7528\u4e2d\u7684\u6570\u636e\u4e00\u81f4\u6027\u4ee5\u53ca\u5185\u5b58\u7684\u4f18\u5316\u975e\u5e38\u91cd\u8981\u3002<\/li>\n<li>\u5982\u679c\u5bb9\u5668\u91cd\u7528\u662f\u6fc0\u6d3b\u72b6\u6001\uff0cjob\u5c06\u4e0d\u4f1a\u91ca\u653e\u5bb9\u5668\u7ed9\u8d44\u6e90\u7ba1\u7406\u5668\uff0c\u9664\u975e\u6240\u6709\u5c5e\u4e8e\u8be5job\u7684\u4efb\u52a1\u90fd\u5df2\u5b8c\u6210\uff0c\u8fd9\u79cd\u60c5\u51b5\u4f1a\u5f71\u54cd\u5230\u516c\u5e73\u8c03\u5ea6\u7684\u539f\u5219\u3002<\/li>\n<\/ul>\n<h3>12.11.2\u00a0\u62a2\u5360\u4efb\u52a1<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u62a2\u5360\u4efb\u52a1\u53ef\u7cbe\u7b80\u961f\u5217\u4e2d\u7684job\u8fd0\u884c\u5e76\u63d0\u9ad8\u8d44\u6e90\u5229\u7528\u7387\uff0c\u7531ResourceManager\u7684capacity scheduler\u5b9e\u73b0\uff0c\u5176\u7b80\u6613\u6d41\u7a0b\u5982\u4e0b\uff1a<\/p>\n<ol>\n<li>\u5047\u8bbe\u5b58\u5728\u4e24\u4e2a\u961f\u5217A\u548cB\u3002\u5176\u4e2d\u961f\u5217A\u7684capacity\u4e3a25%\uff0c\u961f\u5217B\u7684capacity\u4e3a75%\u3002<\/li>\n<li>\u521d\u59cb\u72b6\u6001\u4e0b\uff0c\u4efb\u52a11\u53d1\u9001\u7ed9\u961f\u5217A\uff0c\u6b64\u4efb\u52a1\u9700\u898175%\u7684\u96c6\u7fa4\u8d44\u6e90\u3002\u4e4b\u540e\u4efb\u52a12\u53d1\u9001\u5230\u4e86\u961f\u5217B\uff0c\u6b64\u4efb\u52a1\u9700\u898150%\u7684\u96c6\u7fa4\u8d44\u6e90\u3002<\/li>\n<li>\u4efb\u52a11\u5c06\u4f1a\u4f7f\u7528\u961f\u5217A\u63d0\u4f9b\u768425%\u7684\u96c6\u7fa4\u8d44\u6e90\uff0c\u5e76\u4ece\u961f\u5217B\u83b7\u53d6\u768450%\u7684\u96c6\u7fa4\u8d44\u6e90\u3002\u961f\u5217B\u4fdd\u755925%\u7684\u96c6\u7fa4\u8d44\u6e90\u3002<\/li>\n<li>\u542f\u7528\u62a2\u5360\u4efb\u52a1\u7279\u6027\uff0c\u5219\u4efb\u52a11\u4f7f\u7528\u7684\u8d44\u6e90\u5c06\u4f1a\u88ab\u62a2\u5360\u3002\u961f\u5217B\u4f1a\u4ece\u961f\u5217A\u4e2d\u83b7\u53d625%\u7684\u96c6\u7fa4\u8d44\u6e90\u4ee5\u6ee1\u8db3\u4efb\u52a12\u7684\u6267\u884c\u3002<\/li>\n<li>\u5f53\u4efb\u52a12\u5b8c\u6210\u540e\uff0c\u96c6\u7fa4\u4e2d\u5b58\u5728\u8db3\u591f\u7684\u8d44\u6e90\u65f6\uff0c\u4efb\u52a11\u5c06\u91cd\u65b0\u5f00\u59cb\u6267\u884c\u3002<\/li>\n<\/ol>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; YARN &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<\/p>\n<p>\u886812-43\u00a0Preemption\u914d\u7f6e<\/p>\n<p>yarn.resourcemanager.scheduler.monitor.enable<br \/>\n\u6839\u636e\u201cyarn.resourcemanager.scheduler.monitor.policies\u201d\u4e2d\u7684\u7b56\u7565\uff0c\u542f\u7528\u65b0\u7684scheduler\u76d1\u63a7\u3002\u8bbe\u7f6e\u4e3a\u201ctrue\u201d\u8868\u793a\u542f\u7528\u76d1\u63a7\uff0c\u5e76\u6839\u636escheduler\u7684\u4fe1\u606f\uff0c\u542f\u52a8\u62a2\u5360\u7684\u529f\u80fd\u3002\u8bbe\u7f6e\u4e3a\u201cfalse\u201d\u8868\u793a\u4e0d\u542f\u7528\u3002<br \/>\nfalse<br \/>\nyarn.resourcemanager.scheduler.monitor.policies<br \/>\n\u8bbe\u7f6e\u4e0escheduler\u914d\u5408\u7684\u201cSchedulingEditPolicy\u201d\u7684\u7c7b\u7684\u6e05\u5355\u3002<br \/>\norg.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy<br \/>\nyarn.resourcemanager.monitor.capacity.preemption.observe_only<br \/>\n\u8bbe\u7f6e\u4e3a\u201ctrue\u201d\uff0c\u5219\u6267\u884c\u7b56\u7565\uff0c\u4f46\u662f\u4e0d\u5bf9\u96c6\u7fa4\u8d44\u6e90\u8fdb\u7a0b\u62a2\u5360\u64cd\u4f5c\u3002 \u8bbe\u7f6e\u4e3a\u201cfalse\u201d\uff0c\u5219\u6267\u884c\u7b56\u7565\uff0c\u4e14\u6839\u636e\u7b56\u7565\u542f\u7528\u96c6\u7fa4\u8d44\u6e90\u62a2\u5360\u7684\u529f\u80fd\u3002<br \/>\nfalse<br \/>\nyarn.resourcemanager.monitor.capacity.preemption.monitoring_interval<br \/>\n\u6839\u636e\u7b56\u7565\u76d1\u63a7\u7684\u65f6\u95f4\u95f4\u9694\uff0c\u5355\u4f4d\u4e3a\u6beb\u79d2\u3002<br \/>\n3000<br \/>\nyarn.resourcemanager.monitor.capacity.preemption.max_wait_before_kill<br \/>\n\u5e94\u7528\u53d1\u9001\u62a2\u5360\u9700\u6c42\u5230\u505c\u6b62container\uff08\u91ca\u653e\u8d44\u6e90\uff09\u7684\u65f6\u95f4\u95f4\u9694\uff0c\u5355\u4f4d\u4e3a\u6beb\u79d2\u3002 \u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u82e5ApplicationMaster15\u79d2\u5185\u6ca1\u6709\u7ec8\u6b62container\uff0cResourceManager\u7b49\u5f8515\u79d2\u540e\u4f1a\u5f3a\u5236\u7ec8\u6b62\u3002<br \/>\n15000<br \/>\nyarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round<br \/>\n\u5728\u4e00\u4e2a\u5468\u671f\u5185\u80fd\u591f\u62a2\u5360\u8d44\u6e90\u7684\u6700\u5927\u7684\u6bd4\u4f8b\u3002<br \/>\n0.1<br \/>\nyarn.resourcemanager.monitor.capacity.preemption.max_ignored_over_capacity<br \/>\n\u96c6\u7fa4\u4e2d\u8d44\u6e90\u603b\u91cf\u4e58\u4ee5\u6b64\u914d\u7f6e\u9879\u7684\u503c\u52a0\u4e0a\u67d0\u4e2a\u961f\u5217\uff08\u4f8b\u5982\u961f\u5217A\uff09\u539f\u6709\u7684\u8d44\u6e90\u91cf\u4e3a\u8d44\u6e90\u62a2\u5360\u76f2\u533a\u3002\u5f53\u961f\u5217A\u4e2d\u7684\u4efb\u52a1\u5b9e\u9645\u4f7f\u7528\u7684\u8d44\u6e90\u8d85\u8fc7\u8be5\u62a2\u5360\u76f2\u533a\u65f6\uff0c\u8d85\u8fc7\u90e8\u5206\u7684\u8d44\u6e90\u5c06\u4f1a\u88ab\u62a2\u5360\u3002 \u8bf4\u660e\uff1a \u8bbe\u7f6e\u7684\u503c\u8d8a\u5c0f\u8d8a\u6709\u5229\u4e8e\u8d44\u6e90\u62a2\u5360\u3002<br \/>\n0<br \/>\nyarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor<br \/>\n\u8bbe\u7f6e\u62a2\u5360\u76ee\u6807\uff0cContainer\u53ea\u4f1a\u62a2\u5360\u6240\u914d\u7f6e\u6bd4\u4f8b\u7684\u8d44\u6e90\u3002 \u793a\u4f8b\uff0c\u5982\u679c\u8bbe\u7f6e\u4e3a0.5\uff0c\u5219\u57285*\u201cyarn.resourcemanager.monitor.capacity.preemption.max_wait_before_kill\u201d\u7684\u65f6\u95f4\u5185\uff0c\u4efb\u52a1\u4f1a\u56de\u6536\u6240\u62a2\u5360\u8d44\u6e90\u7684\u8fd195%\u3002\u5373\u63a5\u8fde\u62a2\u53605\u6b21\uff0c\u6bcf\u6b21\u62a2\u5360\u5f85\u62a2\u5360\u8d44\u6e90\u76840.5\uff0c\u5448\u51e0\u4f55\u6536\u655b\uff0c\u6bcf\u6b21\u7684\u65f6\u95f4\u95f4\u9694\u4e3a\u201cyarn.resourcemanager.monitor.capacity.preemption.max_wait_before_kill\u201d\u3002<br \/>\n1<\/p>\n<h3>12.11.3\u00a0\u4efb\u52a1\u4f18\u5148\u7ea7<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u96c6\u7fa4\u7684\u8d44\u6e90\u7ade\u4e89\u573a\u666f\u5982\u4e0b\uff1a<\/p>\n<ol>\n<li>\u63d0\u4ea4\u4e24\u4e2a\u4f4e\u4f18\u5148\u7ea7\u7684\u5e94\u7528Job 1\u548cJob 2\u3002<\/li>\n<li>\u6b63\u5728\u8fd0\u884c\u4e2d\u7684Job 1\u548cJob 2\u6709\u90e8\u5206task\u5904\u4e8erunning\u72b6\u6001\uff0c\u4f46\u7531\u4e8e\u96c6\u7fa4\u6216\u961f\u5217\u8d44\u6e90\u5bb9\u91cf\u6709\u9650\uff0c\u4ecd\u6709\u90e8\u5206task\u672a\u5f97\u5230\u8d44\u6e90\u800c\u5904\u4e8epending\u72b6\u6001\u3002<\/li>\n<li>\u63d0\u4ea4\u4e00\u4e2a\u8f83\u9ad8\u4f18\u5148\u7ea7\u7684\u5e94\u7528Job 3\uff0c\u6b64\u65f6\u4f1a\u51fa\u73b0\u5982\u4e0b\u8d44\u6e90\u5206\u914d\u60c5\u51b5\uff1a\u5f53Job 1\u548cJob 2\u4e2drunning\u72b6\u6001\u7684task\u8fd0\u884c\u7ed3\u675f\u5e76\u91ca\u653e\u8d44\u6e90\u540e\uff0cJob 3\u4e2d\u5904\u4e8epending\u72b6\u6001\u7684task\u5c06\u4f18\u5148\u5f97\u5230\u8fd9\u90e8\u5206\u65b0\u91ca\u653e\u7684\u8d44\u6e90\u3002<\/li>\n<li>Job 3\u5b8c\u6210\u540e\uff0c\u8d44\u6e90\u91ca\u653e\u7ed9Job 1\u3001Job 2\u7ee7\u7eed\u6267\u884c\u3002<\/li>\n<\/ol>\n<p>\u7528\u6237\u53ef\u4ee5\u5728YARN\u4e2d\u914d\u7f6e\u4efb\u52a1\u7684\u4f18\u5148\u7ea7\u3002\u4efb\u52a1\u4f18\u5148\u7ea7\u662f\u901a\u8fc7ResourceManager\u7684Capacity Scheduler\u5b9e\u73b0\u7684\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u8bbe\u7f6e\u53c2\u6570\u201cyarn.app.priority\u201d\u6216\u201cmapreduce.job.priority\u201d\uff0c\u4f7f\u7528\u547d\u4ee4\u884c\u63a5\u53e3\u6216API\u63a5\u53e3\u8bbe\u7f6e\u4efb\u52a1\u4f18\u5148\u7ea7\u3002\u82e5\u4e24\u79cd\u63a5\u53e3\u90fd\u4f7f\u7528\uff0c\u5219\u8003\u8651\u8bbe\u7f6e\u53c2\u6570\u201cyarn.app.priority\u201d\u3002<\/p>\n<ul>\n<li>\u547d\u4ee4\u884c\u63a5\u53e3\u3002<\/li>\n<\/ul>\n<p>\u53d1\u9001\u4efb\u52a1\u65f6\uff0c\u6dfb\u52a0\u201c-Dyarn.app.priority=<!-- -->&lt;priority&gt;<!-- -->\u201d\u53c2\u6570\u3002<\/p>\n<p>&lt;priority&gt;<!-- -->\u53ef\u4ee5\u8bbe\u7f6e\u4e3a\uff1a<\/p>\n<ul>\n<li>VERY_HIGH<!-- -->\n<ul>\n<li>HIGH<\/li>\n<li>NORMAL<\/li>\n<li>LOW<\/li>\n<li>VERY_LOW<\/li>\n<\/ul>\n<\/li>\n<li>API\u63a5\u53e3\u3002<\/li>\n<\/ul>\n<p>\u7528\u6237\u4e5f\u53ef\u4ee5\u4f7f\u7528API\u914d\u7f6e\u5bf9\u8c61\u7684\u4f18\u5148\u7ea7\u3002<\/p>\n<p>\u8bbe\u7f6e\u4f18\u5148\u7ea7\uff0c\u53ef\u901a\u8fc7**Configuration.set(&quot;yarn.app.priority&quot;, <!-- -->&lt;priority&gt;<!-- -->)<strong>\u6216<\/strong>Job.setPriority(JobPriority priority)**\u8bbe\u7f6e\u3002\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>&lt;priority&gt;<!-- --> \u7684\u53ef\u66ff\u6362\u503c\u53c2\u89c1\u2022 \u547d\u4ee4\u884c\u63a5\u53e3\u3002\u3002<\/p>\n<h3>12.11.4\u00a0\u8282\u70b9\u914d\u7f6e\u8c03\u4f18<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5408\u7406\u914d\u7f6e\u5927\u6570\u636e\u96c6\u7fa4\u7684\u8c03\u5ea6\u5668\u540e\uff0c\u8fd8\u53ef\u901a\u8fc7\u8c03\u8282\u6bcf\u4e2a\u8282\u70b9\u7684\u53ef\u7528\u5185\u5b58\u3001CPU\u8d44\u6e90\u53ca\u672c\u5730\u78c1\u76d8\u7684\u914d\u7f6e\u8fdb\u884c\u6027\u80fd\u8c03\u4f18\u3002<\/p>\n<p>\u5177\u4f53\u5305\u62ec\u4ee5\u4e0b\u914d\u7f6e\u9879\uff1a<\/p>\n<ul>\n<li>\u53ef\u7528\u5185\u5b58<\/li>\n<li>CPU\u865a\u62df\u6838\u6570<\/li>\n<li>\u7269\u7406CPU\u4f7f\u7528\u767e\u5206\u6bd4<\/li>\n<li>\u5185\u5b58\u548cCPU\u8d44\u6e90\u7684\u534f\u8c03<\/li>\n<li>\u672c\u5730\u78c1\u76d8<\/li>\n<\/ul>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p>\u53c2\u6570\u5165\u53e3\uff1a<\/p>\n<p>\u5728FusionInsight Manager\u7cfb\u7edf\u4e2d\uff0c\u9009\u62e9\u201c\u670d\u52a1\u7ba1\u7406 &gt; Yarn &gt; \u670d\u52a1\u914d\u7f6e\u201d\uff0c\u201c\u53c2\u6570\u7c7b\u522b\u201d\u7c7b\u578b\u8bbe\u7f6e\u4e3a\u201c\u5168\u90e8\u914d\u7f6e\u201d\u3002\u5728\u641c\u7d22\u6846\u4e2d\u8f93\u5165\u53c2\u6570\u540d\u79f0\u3002<\/p>\n<ul>\n<li><strong>\u53ef\u7528\u5185\u5b58<\/strong><\/li>\n<\/ul>\n<p>\u9664\u4e86\u5206\u914d\u7ed9\u64cd\u4f5c\u7cfb\u7edf\u3001\u5176\u4ed6\u670d\u52a1\u7684\u5185\u5b58\u5916\uff0c\u5269\u4f59\u7684\u8d44\u6e90\u5e94\u5c3d\u91cf\u5206\u914d\u7ed9YARN\u3002\u901a\u8fc7\u5982\u4e0b\u914d\u7f6e\u53c2\u6570\u8fdb\u884c\u8c03\u6574\u3002<\/p>\n<p>\u4f8b\u5982\uff0c\u5982\u679c\u4e00\u4e2acontainer\u9ed8\u8ba4\u4f7f\u7528512M\uff0c\u5219\u5185\u5b58\u4f7f\u7528\u7684\u8ba1\u7b97\u516c\u5f0f\u4e3a\uff1a512M*container\u6570\u3002<\/p>\n<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0cMap\u6216Reduce container\u4f1a\u4f7f\u75281\u4e2a\u865a\u62dfCPU\u5185\u6838\u548c1024MB\u5185\u5b58\uff0cApplicationMaster\u4f7f\u75281536MB\u5185\u5b58\u3002<\/p>\n<p>yarn.nodemanager.resource.memory-mb<br \/>\n\u8bbe\u7f6e\u53ef\u5206\u914d\u7ed9\u5bb9\u5668\u7684\u7269\u7406\u5185\u5b58\u6570\u91cf\u3002\u5efa\u8bae\u914d\u7f6e\u4e3a&gt;24576\uff0824G\uff09 \u5355\u4f4d\uff1aMB<br \/>\n8192<\/p>\n<ul>\n<li><strong>CPU\u865a\u62df\u6838\u6570<\/strong><\/li>\n<\/ul>\n<p>\u5efa\u8bae\u5c06\u6b64\u914d\u7f6e\u8bbe\u5b9a\u5728\u903b\u8f91\u6838\u6570\u76841.5\uff5e2\u500d\u4e4b\u95f4\u3002\u5982\u679c\u4e0a\u5c42\u8ba1\u7b97\u5e94\u7528\u5bf9CPU\u7684\u8ba1\u7b97\u80fd\u529b\u8981\u6c42\u4e0d\u9ad8\uff0c\u53ef\u4ee5\u914d\u7f6e\u4e3a2\u500d\u7684\u903b\u8f91CPU\u3002<\/p>\n<p>yarn.nodemanager.resource.cpu-vcores<br \/>\n\u8868\u793a\u8be5\u8282\u70b9\u4e0aYARN\u53ef\u4f7f\u7528\u7684\u865a\u62dfCPU\u4e2a\u6570\uff0c\u9ed8\u8ba4\u662f8\u3002 \u76ee\u524d\u63a8\u8350\u5c06\u8be5\u503c\u8bbe\u503c\u4e3a\u903b\u8f91CPU\u6838\u6570\u76841.5\uff5e2\u500d\u4e4b\u95f4\u3002<br \/>\n8<\/p>\n<ul>\n<li><strong>\u7269\u7406CPU\u4f7f\u7528\u767e\u5206\u6bd4<\/strong><\/li>\n<\/ul>\n<p>\u5efa\u8bae\u9884\u7559\u9002\u91cf\u7684CPU\u7ed9\u64cd\u4f5c\u7cfb\u7edf\u548c\u5176\u4ed6\u8fdb\u7a0b\uff08\u6570\u636e\u5e93\u3001HBase\u7b49\uff09\u5916\uff0c\u5269\u4f59\u7684CPU\u6838\u90fd\u5206\u914d\u7ed9YARN\u3002\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u914d\u7f6e\u53c2\u6570\u8fdb\u884c\u8c03\u6574\u3002<\/p>\n<p>yarn.nodemanager.resource.percentage-physical-cpu-limit<br \/>\n\u8868\u793a\u8be5\u8282\u70b9\u4e0aYARN\u53ef\u4f7f\u7528\u7684\u7269\u7406CPU\u767e\u5206\u6bd4\u3002\u9ed8\u8ba4\u662f100\uff0c\u5373\u4e0d\u8fdb\u884cCPU\u63a7\u5236\uff0cYARN\u53ef\u4ee5\u4f7f\u7528\u8282\u70b9\u5168\u90e8CPU\u3002\u8be5\u53c2\u6570\u53ea\u652f\u6301\u67e5\u770b\uff0c\u53ef\u901a\u8fc7\u8c03\u6574YARN\u7684RES_CPUSET_PERCENTAGE\u53c2\u6570\u6765\u4fee\u6539\u672c\u53c2\u6570\u503c\u3002\u6ce8\u610f\uff0c\u76ee\u524d\u63a8\u8350\u5c06\u8be5\u503c\u8bbe\u4e3a\u53ef\u4f9bYARN\u96c6\u7fa4\u4f7f\u7528\u7684CPU\u767e\u5206\u6570\u3002 \u4f8b\u5982\uff1a\u5f53\u524d\u8282\u70b9\u9664\u4e86YARN\u670d\u52a1\u5916\u7684\u5176\u4ed6\u670d\u52a1\uff08\u5982HBase\u3001HDFS\u3001Hive\u7b49\uff09\u53ca\u7cfb\u7edf\u8fdb\u7a0b\u4f7f\u7528CPU\u4e3a20%\u5de6\u53f3\uff0c\u5219\u53ef\u4ee5\u4f9bYARN\u8c03\u5ea6\u7684CPU\u4e3a1-20%=80%\uff0c\u5373\u914d\u7f6e\u6b64\u53c2\u6570\u4e3a80\u3002<br \/>\n90<\/p>\n<ul>\n<li><strong>\u672c\u5730\u78c1\u76d8<\/strong><\/li>\n<\/ul>\n<p>\u7531\u4e8e\u672c\u5730\u78c1\u76d8\u4f1a\u63d0\u4f9b\u7ed9MapReduce\u5199job\u6267\u884c\u7684\u4e2d\u95f4\u7ed3\u679c\uff0c\u6570\u636e\u91cf\u5927\u3002\u56e0\u6b64\u914d\u7f6e\u7684\u539f\u5219\u662f\u78c1\u76d8\u5c3d\u91cf\u591a\uff0c\u4e14\u78c1\u76d8\u7a7a\u95f4\u5c3d\u91cf\u5927\uff0c\u5355\u4e2a\u8fbe\u5230\u767eGB\u4ee5\u4e0a\u89c4\u6a21\u6700\u597d\u3002\u7b80\u5355\u7684\u505a\u6cd5\u662f\u914d\u7f6e\u548cdata node\u76f8\u540c\u7684\u78c1\u76d8\uff0c\u53ea\u5728\u6700\u4e0b\u4e00\u7ea7\u76ee\u5f55\u4e0a\u4e0d\u540c\u5373\u53ef\u3002<\/p>\n<p>\u00a0\u8bf4\u660e\uff1a<\/p>\n<p>\u591a\u4e2a\u78c1\u76d8\u4e4b\u95f4\u4f7f\u7528\u9017\u53f7\u9694\u5f00\u3002<\/p>\n<p>yarn.nodemanager.log-dirs<br \/>\n\u65e5\u5fd7\u5b58\u653e\u5730\u5740\uff08\u53ef\u914d\u7f6e\u591a\u4e2a\u76ee\u5f55\uff09\u3002 \u5bb9\u5668\u65e5\u5fd7\u7684\u5b58\u50a8\u4f4d\u7f6e\u3002\u9ed8\u8ba4\u503c\u4e3a%{@auto.detect.datapart.nm.logs}\u3002\u5982\u679c\u6709\u6570\u636e\u5206\u533a\uff0c\u57fa\u4e8e\u8be5\u6570\u636e\u5206\u533a\u751f\u6210\u4e00\u4e2a\u7c7b\u4f3c\/srv\/BigData\/hadoop\/data1\/nm\/containerlogs,\/srv\/BigData\/hadoop\/data2\/nm\/containerlogs\u7684\u8def\u5f84\u6e05\u5355\u3002\u5982\u679c\u6ca1\u6709\u6570\u636e\u5206\u533a\uff0c\u751f\u6210\u9ed8\u8ba4\u8def\u5f84\/srv\/BigData\/yarn\/data1\/nm\/containerlogs\u3002\u9664\u4e86\u4f7f\u7528\u8868\u8fbe\u5f0f\u4ee5\u5916\uff0c\u8fd8\u53ef\u4ee5\u8f93\u5165\u5b8c\u6574\u7684\u8def\u5f84\u6e05\u5355\uff0c\u6bd4\u5982\/srv\/BigData\/yarn\/data1\/nm\/containerlogs\u6216\/srv\/BigData\/yarn\/data1\/nm\/containerlogs,\/srv\/BigData\/yarn\/data2\/nm\/containerlogs\u3002\u8fd9\u6837\u6570\u636e\u5c31\u4f1a\u5b58\u50a8\u5728\u6240\u6709\u8bbe\u7f6e\u7684\u76ee\u5f55\u4e2d\uff0c\u4e00\u822c\u4f1a\u662f\u5728\u4e0d\u540c\u7684\u8bbe\u5907\u4e2d\u3002\u4e3a\u4fdd\u8bc1\u78c1\u76d8IO\u8d1f\u8f7d\u5747\u8861\uff0c\u6700\u597d\u63d0\u4f9b\u51e0\u4e2a\u8def\u5f84\u4e14\u6bcf\u4e2a\u8def\u5f84\u90fd\u5bf9\u5e94\u4e00\u4e2a\u5355\u72ec\u7684\u78c1\u76d8\u3002\u5e94\u7528\u7a0b\u5e8f\u7684\u672c\u5730\u5316\u540e\u7684\u65e5\u5fd7\u76ee\u5f55\u5b58\u5728\u4e8e\u76f8\u5bf9\u8def\u5f84\/application_%{appid}\u4e2d\u3002\u5355\u72ec\u5bb9\u5668\u7684\u65e5\u5fd7\u76ee\u5f55\uff0c\u5373container_{$contid}\uff0c\u662f\u8be5\u8def\u5f84\u4e0b\u7684\u5b50\u76ee\u5f55\u3002\u6bcf\u4e2a\u5bb9\u5668\u76ee\u5f55\u90fd\u542b\u5bb9\u5668\u751f\u6210\u7684stderr\u3001stdin\u53casyslog\u6587\u4ef6\u3002\u8981\u65b0\u589e\u76ee\u5f55\uff0c\u6bd4\u5982\u65b0\u589e\/srv\/BigData\/yarn\/data2\/nm\/containerlogs\u76ee\u5f55\uff0c\u5e94\u9996\u5148\u5220\u9664\/srv\/BigData\/yarn\/data2\/nm\/containerlogs\u4e0b\u7684\u6587\u4ef6\u3002\u4e4b\u540e\uff0c\u4e3a\/srv\/BigData\/yarn\/data2\/nm\/containerlogs\u8d4b\u4e88\u8ddf\/srv\/BigData\/yarn\/data1\/nm\/containerlogs\u4e00\u6837\u7684\u8bfb\u5199\u6743\u9650\uff0c\u518d\u5c06\/srv\/BigData\/yarn\/data1\/nm\/containerlogs\u4fee\u6539\u4e3a\/srv\/BigData\/yarn\/data1\/nm\/containerlogs,\/srv\/BigData\/yarn\/data2\/nm\/containerlogs\u3002\u53ef\u4ee5\u65b0\u589e\u76ee\u5f55\uff0c\u4f46\u4e0d\u8981\u4fee\u6539\u6216\u5220\u9664\u73b0\u6709\u76ee\u5f55\u3002\u5426\u5219\uff0cNodeManager\u7684\u6570\u636e\u5c06\u4e22\u5931\uff0c\u4e14\u670d\u52a1\u5c06\u4e0d\u53ef\u7528\u3002 \u3010\u9ed8\u8ba4\u503c\u3011%{@auto.detect.datapart.nm.logs} \u3010\u6ce8\u610f\u3011\u8bf7\u8c28\u614e\u4fee\u6539\u8be5\u9879\u3002\u5982\u679c\u914d\u7f6e\u4e0d\u5f53\uff0c\u5c06\u9020\u6210\u670d\u52a1\u4e0d\u53ef\u7528\u3002\u5f53\u89d2\u8272\u7ea7\u522b\u7684\u8be5\u914d\u7f6e\u9879\u4fee\u6539\u540e\uff0c\u6240\u6709\u5b9e\u4f8b\u7ea7\u522b\u7684\u8be5\u914d\u7f6e\u9879\u90fd\u5c06\u88ab\u4fee\u6539\u3002\u5982\u679c\u5b9e\u4f8b\u7ea7\u522b\u7684\u914d\u7f6e\u9879\u4fee\u6539\u540e\uff0c\u5176\u4ed6\u5b9e\u4f8b\u7684\u8be5\u914d\u7f6e\u9879\u7684\u503c\u4fdd\u6301\u4e0d\u53d8\u3002<br \/>\n%{@auto.detect.datapart.nm.logs}<br \/>\nyarn.nodemanager.local-dirs<br \/>\n\u672c\u5730\u5316\u540e\u7684\u6587\u4ef6\u7684\u5b58\u50a8\u4f4d\u7f6e\u3002\u9ed8\u8ba4\u503c\u4e3a%{@auto.detect.datapart.nm.localdir}\u3002\u5982\u679c\u6709\u6570\u636e\u5206\u533a\uff0c\u57fa\u4e8e\u8be5\u6570\u636e\u5206\u533a\u751f\u6210\u4e00\u4e2a\u7c7b\u4f3c\/srv\/BigData\/hadoop\/data1\/nm\/localdir,\/srv\/BigData\/hadoop\/data2\/nm\/localdir\u7684\u8def\u5f84\u6e05\u5355\u3002\u5982\u679c\u6ca1\u6709\u6570\u636e\u5206\u533a\uff0c\u751f\u6210\u9ed8\u8ba4\u8def\u5f84\/srv\/BigData\/yarn\/data1\/nm\/localdir\u3002\u9664\u4e86\u4f7f\u7528\u8868\u8fbe\u5f0f\u4ee5\u5916\uff0c\u8fd8\u53ef\u4ee5\u8f93\u5165\u5b8c\u6574\u7684\u8def\u5f84\u6e05\u5355\uff0c\u6bd4\u5982\/srv\/BigData\/yarn\/data1\/nm\/localdir\u6216\/srv\/BigData\/yarn\/data1\/nm\/localdir,\/srv\/BigData\/yarn\/data2\/nm\/localdir\u3002\u8fd9\u6837\u6570\u636e\u5c31\u4f1a\u5b58\u50a8\u5728\u6240\u6709\u8bbe\u7f6e\u7684\u76ee\u5f55\u4e2d\uff0c\u4e00\u822c\u4f1a\u662f\u5728\u4e0d\u540c\u7684\u8bbe\u5907\u4e2d\u3002\u4e3a\u4fdd\u8bc1\u78c1\u76d8IO\u8d1f\u8f7d\u5747\u8861\uff0c\u6700\u597d\u63d0\u4f9b\u51e0\u4e2a\u8def\u5f84\u4e14\u6bcf\u4e2a\u8def\u5f84\u90fd\u5bf9\u5e94\u4e00\u4e2a\u5355\u72ec\u7684\u78c1\u76d8\u3002\u5e94\u7528\u7a0b\u5e8f\u7684\u672c\u5730\u5316\u540e\u7684\u6587\u4ef6\u76ee\u5f55\u5b58\u5728\u4e8e\u76f8\u5bf9\u8def\u5f84\/usercache\/%{user}\/appcache\/application_%{appid}\u4e2d\u3002\u5355\u72ec\u5bb9\u5668\u7684\u5de5\u4f5c\u76ee\u5f55\uff0c\u5373container_%{contid}\uff0c\u662f\u8be5\u8def\u5f84\u4e0b\u7684\u5b50\u76ee\u5f55\u3002\u8981\u65b0\u589e\u76ee\u5f55\uff0c\u6bd4\u5982\u65b0\u589e\/srv\/BigData\/yarn\/data2\/nm\/localdir\u76ee\u5f55\uff0c\u5e94\u9996\u5148\u5220\u9664\/srv\/BigData\/yarn\/data2\/nm\/localdir\u4e0b\u7684\u6587\u4ef6\u3002\u4e4b\u540e\uff0c\u4e3a\/srv\/BigData\/hadoop\/data2\/nm\/localdir\u8d4b\u4e88\u8ddf\/srv\/BigData\/hadoop\/data1\/nm\/localdir\u4e00\u6837\u7684\u8bfb\u5199\u6743\u9650\uff0c\u518d\u5c06\/srv\/BigData\/yarn\/data1\/nm\/localdir\u4fee\u6539\u4e3a\/srv\/BigData\/yarn\/data1\/nm\/localdir,\/srv\/BigData\/yarn\/data2\/nm\/localdir\u3002\u53ef\u4ee5\u65b0\u589e\u76ee\u5f55\uff0c\u4f46\u4e0d\u8981\u4fee\u6539\u6216\u5220\u9664\u73b0\u6709\u76ee\u5f55\u3002\u5426\u5219\uff0cNodeManager\u7684\u6570\u636e\u5c06\u4e22\u5931\uff0c\u4e14\u670d\u52a1\u5c06\u4e0d\u53ef\u7528\u3002 \u3010\u9ed8\u8ba4\u503c\u3011%{@auto.detect.datapart.nm.localdir} \u3010\u6ce8\u610f\u3011\u8bf7\u8c28\u614e\u4fee\u6539\u8be5\u9879\u3002\u5982\u679c\u914d\u7f6e\u4e0d\u5f53\uff0c\u5c06\u9020\u6210\u670d\u52a1\u4e0d\u53ef\u7528\u3002\u5f53\u89d2\u8272\u7ea7\u522b\u7684\u8be5\u914d\u7f6e\u9879\u4fee\u6539\u540e\uff0c\u6240\u6709\u5b9e\u4f8b\u7ea7\u522b\u7684\u8be5\u914d\u7f6e\u9879\u90fd\u5c06\u88ab\u4fee\u6539\u3002\u5982\u679c\u5b9e\u4f8b\u7ea7\u522b\u7684\u914d\u7f6e\u9879\u4fee\u6539\u540e\uff0c\u5176\u4ed6\u5b9e\u4f8b\u7684\u8be5\u914d\u7f6e\u9879\u7684\u503c\u4fdd\u6301\u4e0d\u53d8\u3002<br \/>\n%{@auto.detect.datapart.nm.localdir}<\/p>\n<h3>12.11.5\u00a0JVM\u53c2\u6570\u4f18\u5316<\/h3>\n<p>\u64cd\u4f5c\u573a\u666f<\/p>\n<p>\u5f53\u96c6\u7fa4\u6570\u636e\u91cf\u8fbe\u5230\u4e00\u5b9a\u89c4\u6a21\u540e\uff0cJVM\u7684\u9ed8\u8ba4\u914d\u7f6e\u5c06\u65e0\u6cd5\u6ee1\u8db3\u96c6\u7fa4\u7684\u4e1a\u52a1\u9700\u6c42\uff0c\u8f7b\u5219\u96c6\u7fa4\u53d8\u6162\uff0c\u91cd\u5219\u96c6\u7fa4\u670d\u52a1\u4e0d\u53ef\u7528\u3002\u6240\u4ee5\u9700\u8981\u6839\u636e\u5b9e\u9645\u7684\u4e1a\u52a1\u60c5\u51b5\u8fdb\u884c\u5408\u7406\u7684JVM\u53c2\u6570\u914d\u7f6e\uff0c\u63d0\u9ad8\u96c6\u7fa4\u6027\u80fd\u3002<\/p>\n<p>\u64cd\u4f5c\u6b65\u9aa4<\/p>\n<p><strong>\u53c2\u6570\u5165\u53e3\uff1a<\/strong><\/p>\n<p>Yarn\u89d2\u8272\u76f8\u5173\u7684JVM\u53c2\u6570\u9700\u8981\u914d\u7f6e\u5728\u201c${HADOOP_HOME}\/etc\/hadoop\u201d\u76ee\u5f55\u4e0b\u7684\u201cyarn-env.sh\u201d\u6587\u4ef6\u4e2d\u3002<\/p>\n<p>\u6bcf\u4e2a\u89d2\u8272\u90fd\u6709\u5404\u81ea\u7684JVM\u53c2\u6570\u914d\u7f6e\u53d8\u91cf\uff0c\u5982\u886812-44\u3002<\/p>\n<p>\u886812-44\u00a0Yarn\u76f8\u5173JVM\u53c2\u6570\u914d\u7f6e\u53d8\u91cf<\/p>\n<p>YARN_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdYarn\u7684\u6240\u6709\u89d2\u8272\u3002<br \/>\nYARN_CLIENT_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdYarn\u7684Client\u8fdb\u7a0b\u3002<br \/>\nYARN_RESOURCEMANAGER_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdYarn\u7684ResourceManager\u3002<br \/>\nYARN_HISTORYSERVER_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdYarn\u7684HistoryServer\u3002<br \/>\nYARN_TIMELINESERVER_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdYarn\u7684TimelineServer\u3002<br \/>\nYARN_NODEMANAGER_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdYarn\u7684NodeManager\u3002<br \/>\nYARN_PROXYSERVER_OPTS<br \/>\n\u8be5\u53d8\u91cf\u4e2d\u8bbe\u7f6e\u7684\u53c2\u6570\uff0c\u5c06\u5f71\u54cdYarn\u7684ProxyServer\u3002<\/p>\n<p><strong>\u914d\u7f6e\u65b9\u5f0f\u4e3e\u4f8b\uff1a<\/strong><\/p>\n<pre>\n\n<div class=\"codecolorer-container text solarized-dark language-bash\" style=\"overflow:auto;white-space:nowrap;width:800px;\"><table cellspacing=\"0\" cellpadding=\"0\"><tbody><tr><td class=\"line-numbers\"><div>1<br \/>2<br \/>3<br \/><\/div><\/td><td><div class=\"text codecolorer\">1<br \/>\n2export HADOOP_NAMENODE_OPTS=&amp;quot;-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS&amp;quot;<br \/>\n3<\/div><\/td><\/tr><\/tbody><\/table><\/div>\n\n<\/pre>\n","protected":false},"excerpt":{"rendered":"<p>\u76ee \u5f55 12 \u6027\u80fd\u8c03\u4f18 12.1 \u914d\u7f6e\u539f\u5219 12.2 Manager 12.2.1 \u63d0\u5347Manager\u914d\u7f6e\u670d\u52a1\u53c2\u6570\u7684\u6548\u7387 12.2.2 \u6839\u636e\u96c6\u7fa4\u8282\u70b9\u6570\u4f18\u5316Manager\u914d\u7f6e 12.3 HBase 12.3.1 \u63d0\u5347BulkLoad\u6548\u7387 12.3.2 \u63d0\u5347\u8fde\u7eedput\u573a\u666f\u6027\u80fd 12.3.3 Put\u548cScan\u6027\u80fd\u7efc\u5408\u8c03\u4f18 12.3.4 \u63d0\u5347\u5b9e\u65f6\u5199\u6570\u636e\u6548\u7387 12.3.5 \u63d0\u5347\u5b9e\u65f6\u8bfb\u6570\u636e\u6548\u7387 12.3.6 JVM\u53c2\u6570\u4f18\u5316 12.4 HDFS 12.4.1 \u63d0\u5347\u5199\u6027\u80fd 12.4.2 JVM\u53c2\u6570\u4f18\u5316 12.4.3 \u4f7f\u7528\u5ba2\u6237\u7aef\u5143\u6570<\/p>\n","protected":false},"author":1,"featured_media":6522,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_bbp_topic_count":0,"_bbp_reply_count":0,"_bbp_total_topic_count":0,"_bbp_total_reply_count":0,"_bbp_voice_count":0,"_bbp_anonymous_reply_count":0,"_bbp_topic_count_hidden":0,"_bbp_reply_count_hidden":0,"_bbp_forum_subforum_count":0,"footnotes":""},"categories":[2],"tags":[292,17,530,23,26,25,498,534,21,32,10,22,500,190,447,135],"collection":[],"_links":{"self":[{"href":"https:\/\/aqzt.com\/wp-json\/wp\/v2\/posts\/24604"}],"collection":[{"href":"https:\/\/aqzt.com\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/aqzt.com\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/aqzt.com\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/aqzt.com\/wp-json\/wp\/v2\/comments?post=24604"}],"version-history":[{"count":0,"href":"https:\/\/aqzt.com\/wp-json\/wp\/v2\/posts\/24604\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/aqzt.com\/wp-json\/wp\/v2\/media\/6522"}],"wp:attachment":[{"href":"https:\/\/aqzt.com\/wp-json\/wp\/v2\/media?parent=24604"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/aqzt.com\/wp-json\/wp\/v2\/categories?post=24604"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/aqzt.com\/wp-json\/wp\/v2\/tags?post=24604"},{"taxonomy":"collection","embeddable":true,"href":"https:\/\/aqzt.com\/wp-json\/wp\/v2\/collection?post=24604"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}