From c272dbec7ebd18f810ecdd13c63cd0c767b4ed4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?F=E5=98=89=E9=98=B3?= <fjy8018@qq.com>
Date: Wed, 18 Apr 2018 22:19:51 +0800
Subject: [PATCH] =?UTF-8?q?=E4=BD=BF=E7=94=A8Combiner=E5=8A=A0=E5=BF=AB?=
 =?UTF-8?q?=E5=A4=84=E7=90=86=E6=95=88=E7=8E=87?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .idea/workspace.xml                           | 69 +++++++++++--------
 pom.xml                                       |  2 +-
 .../fjy/hadoop/mapreduce/WordCountApp.java    |  4 ++
 3 files changed, 44 insertions(+), 31 deletions(-)

diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 02c16e1..e128e88 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -3,6 +3,7 @@
   <component name="ChangeListManager">
     <list default="true" id="8ccbe0a5-cdf8-4091-a84b-916013277899" name="Default" comment="">
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
+      <change beforePath="$PROJECT_DIR$/pom.xml" afterPath="$PROJECT_DIR$/pom.xml" />
       <change beforePath="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java" afterPath="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java" />
     </list>
     <ignored path="$PROJECT_DIR$/out/" />
@@ -19,8 +20,8 @@
       <file leaf-file-name="pom.xml" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/pom.xml">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="782">
-              <caret line="34" column="40" lean-forward="false" selection-start-line="34" selection-start-column="40" selection-end-line="34" selection-end-column="40" />
+            <state relative-caret-position="138">
+              <caret line="6" column="14" lean-forward="false" selection-start-line="6" selection-start-column="14" selection-end-line="6" selection-end-column="14" />
               <folding />
             </state>
           </provider>
@@ -29,7 +30,7 @@
       <file leaf-file-name="HDFSApp.java" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="161">
+            <state relative-caret-position="23">
               <caret line="19" column="25" lean-forward="false" selection-start-line="19" selection-start-column="25" selection-end-line="19" selection-end-column="25" />
               <folding>
                 <element signature="imports" expanded="true" />
@@ -41,8 +42,8 @@
       <file leaf-file-name="WordCountApp.java" pinned="false" current-in-tab="true">
         <entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="437">
-              <caret line="95" column="68" lean-forward="false" selection-start-line="95" selection-start-column="68" selection-end-line="95" selection-end-column="68" />
+            <state relative-caret-position="755">
+              <caret line="135" column="0" lean-forward="true" selection-start-line="135" selection-start-column="0" selection-end-line="135" selection-end-column="0" />
               <folding>
                 <element signature="imports" expanded="true" />
               </folding>
@@ -122,8 +123,8 @@
   <component name="IdeDocumentHistory">
     <option name="CHANGED_PATHS">
       <list>
-        <option value="$PROJECT_DIR$/pom.xml" />
         <option value="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java" />
+        <option value="$PROJECT_DIR$/pom.xml" />
         <option value="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java" />
       </list>
     </option>
@@ -505,7 +506,7 @@
       <workItem from="1523322335998" duration="659000" />
       <workItem from="1523606327796" duration="135000" />
       <workItem from="1523931905005" duration="6215000" />
-      <workItem from="1524056118416" duration="1130000" />
+      <workItem from="1524056118416" duration="3187000" />
     </task>
     <task id="LOCAL-00001" summary="建立初始工程">
       <created>1521904777876</created>
@@ -542,7 +543,14 @@
       <option name="project" value="LOCAL" />
       <updated>1523940613235</updated>
     </task>
-    <option name="localTasksCounter" value="6" />
+    <task id="LOCAL-00006" summary="工作前自动删除已有文件">
+      <created>1524058535951</created>
+      <option name="number" value="00006" />
+      <option name="presentableId" value="LOCAL-00006" />
+      <option name="project" value="LOCAL" />
+      <updated>1524058535951</updated>
+    </task>
+    <option name="localTasksCounter" value="7" />
     <servers />
   </component>
   <component name="TestHistory">
@@ -578,7 +586,7 @@
     </history-entry>
   </component>
   <component name="TimeTrackingManager">
-    <option name="totallyTimeSpent" value="28137000" />
+    <option name="totallyTimeSpent" value="30194000" />
   </component>
   <component name="ToolWindowManager">
     <frame x="-8" y="-8" width="1936" height="1056" extended-state="6" />
@@ -625,7 +633,8 @@
     <MESSAGE value="JavaAPI基础操作单元测试编写并通过" />
     <MESSAGE value="单节点MapReduce开发WordCount" />
     <MESSAGE value="补全注释" />
-    <option name="LAST_COMMIT_MESSAGE" value="补全注释" />
+    <MESSAGE value="工作前自动删除已有文件" />
+    <option name="LAST_COMMIT_MESSAGE" value="工作前自动删除已有文件" />
   </component>
   <component name="XDebuggerManager">
     <breakpoint-manager />
@@ -918,14 +927,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/pom.xml">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="782">
-          <caret line="34" column="40" lean-forward="false" selection-start-line="34" selection-start-column="40" selection-end-line="34" selection-end-column="40" />
-          <folding />
-        </state>
-      </provider>
-    </entry>
     <entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-common/2.6.0-cdh5.7.0/hadoop-common-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/fs/FileSystem.class">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="455">
@@ -943,16 +944,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="161">
-          <caret line="19" column="25" lean-forward="false" selection-start-line="19" selection-start-column="25" selection-end-line="19" selection-end-column="25" />
-          <folding>
-            <element signature="imports" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
     <entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Mapper.class">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="184">
@@ -991,10 +982,28 @@
         </state>
       </provider>
     </entry>
+    <entry file="file://$PROJECT_DIR$/pom.xml">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="138">
+          <caret line="6" column="14" lean-forward="false" selection-start-line="6" selection-start-column="14" selection-end-line="6" selection-end-column="14" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="23">
+          <caret line="19" column="25" lean-forward="false" selection-start-line="19" selection-start-column="25" selection-end-line="19" selection-end-column="25" />
+          <folding>
+            <element signature="imports" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
     <entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="437">
-          <caret line="95" column="68" lean-forward="false" selection-start-line="95" selection-start-column="68" selection-end-line="95" selection-end-column="68" />
+        <state relative-caret-position="755">
+          <caret line="135" column="0" lean-forward="true" selection-start-line="135" selection-start-column="0" selection-end-line="135" selection-end-column="0" />
           <folding>
             <element signature="imports" expanded="true" />
           </folding>
diff --git a/pom.xml b/pom.xml
index fd500f3..6137ba9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
   <groupId>com.fjy.hadoop</groupId>
   <artifactId>hadoopstudy</artifactId>
-  <version>1.0-SNAPSHOT</version>
+  <version>0.3</version>
   <packaging>jar</packaging>
 
   <name>hadoopstudy</name>
diff --git a/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java b/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java
index c75ed56..20e6953 100644
--- a/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java
+++ b/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java
@@ -119,6 +119,10 @@ public class WordCountApp {
         job.setOutputKeyClass(Text.class);
         job.setOutputValueClass(LongWritable.class);
 
+        //通过job设置combiner处理类，逻辑上与reduce一致，注意，如果要计算平均数等不能使用Combiner!
+        job.setCombinerClass(MyReducer.class);
+
+
         //设置作业处理输出结果的输出路径
         FileOutputFormat.setOutputPath(job, new Path(args[1]));