使用Combiner加快处理效率

2018-04-18 22:19:51 +08:00
parent 71fb47da05
commit c272dbec7e
3 changed files with 44 additions and 31 deletions
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -3,6 +3,7 @@
  <component name="ChangeListManager">
    <list default="true" id="8ccbe0a5-cdf8-4091-a84b-916013277899" name="Default" comment="">
      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
+      <change beforePath="$PROJECT_DIR$/pom.xml" afterPath="$PROJECT_DIR$/pom.xml" />
      <change beforePath="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java" afterPath="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java" />
    </list>
    <ignored path="$PROJECT_DIR$/out/" />
@@ -19,8 +20,8 @@
      <file leaf-file-name="pom.xml" pinned="false" current-in-tab="false">
        <entry file="file://$PROJECT_DIR$/pom.xml">
          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="782">
-              <caret line="34" column="40" lean-forward="false" selection-start-line="34" selection-start-column="40" selection-end-line="34" selection-end-column="40" />
+            <state relative-caret-position="138">
+              <caret line="6" column="14" lean-forward="false" selection-start-line="6" selection-start-column="14" selection-end-line="6" selection-end-column="14" />
              <folding />
            </state>
          </provider>
@@ -29,7 +30,7 @@
      <file leaf-file-name="HDFSApp.java" pinned="false" current-in-tab="false">
        <entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="161">
+            <state relative-caret-position="23">
              <caret line="19" column="25" lean-forward="false" selection-start-line="19" selection-start-column="25" selection-end-line="19" selection-end-column="25" />
              <folding>
                <element signature="imports" expanded="true" />
@@ -41,8 +42,8 @@
      <file leaf-file-name="WordCountApp.java" pinned="false" current-in-tab="true">
        <entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java">
          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="437">
-              <caret line="95" column="68" lean-forward="false" selection-start-line="95" selection-start-column="68" selection-end-line="95" selection-end-column="68" />
+            <state relative-caret-position="755">
+              <caret line="135" column="0" lean-forward="true" selection-start-line="135" selection-start-column="0" selection-end-line="135" selection-end-column="0" />
              <folding>
                <element signature="imports" expanded="true" />
              </folding>
@@ -122,8 +123,8 @@
  <component name="IdeDocumentHistory">
    <option name="CHANGED_PATHS">
      <list>
-        <option value="$PROJECT_DIR$/pom.xml" />
        <option value="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java" />
+        <option value="$PROJECT_DIR$/pom.xml" />
        <option value="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java" />
      </list>
    </option>
@@ -505,7 +506,7 @@
      <workItem from="1523322335998" duration="659000" />
      <workItem from="1523606327796" duration="135000" />
      <workItem from="1523931905005" duration="6215000" />
-      <workItem from="1524056118416" duration="1130000" />
+      <workItem from="1524056118416" duration="3187000" />
    </task>
    <task id="LOCAL-00001" summary="建立初始工程">
      <created>1521904777876</created>
@@ -542,7 +543,14 @@
      <option name="project" value="LOCAL" />
      <updated>1523940613235</updated>
    </task>
-    <option name="localTasksCounter" value="6" />
+    <task id="LOCAL-00006" summary="工作前自动删除已有文件">
+      <created>1524058535951</created>
+      <option name="number" value="00006" />
+      <option name="presentableId" value="LOCAL-00006" />
+      <option name="project" value="LOCAL" />
+      <updated>1524058535951</updated>
+    </task>
+    <option name="localTasksCounter" value="7" />
    <servers />
  </component>
  <component name="TestHistory">
@@ -578,7 +586,7 @@
    </history-entry>
  </component>
  <component name="TimeTrackingManager">
-    <option name="totallyTimeSpent" value="28137000" />
+    <option name="totallyTimeSpent" value="30194000" />
  </component>
  <component name="ToolWindowManager">
    <frame x="-8" y="-8" width="1936" height="1056" extended-state="6" />
@@ -625,7 +633,8 @@
    <MESSAGE value="JavaAPI基础操作单元测试编写并通过" />
    <MESSAGE value="单节点MapReduce开发WordCount" />
    <MESSAGE value="补全注释" />
-    <option name="LAST_COMMIT_MESSAGE" value="补全注释" />
+    <MESSAGE value="工作前自动删除已有文件" />
+    <option name="LAST_COMMIT_MESSAGE" value="工作前自动删除已有文件" />
  </component>
  <component name="XDebuggerManager">
    <breakpoint-manager />
@@ -918,14 +927,6 @@
        </state>
      </provider>
    </entry>
-    <entry file="file://$PROJECT_DIR$/pom.xml">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="782">
-          <caret line="34" column="40" lean-forward="false" selection-start-line="34" selection-start-column="40" selection-end-line="34" selection-end-column="40" />
-          <folding />
-        </state>
-      </provider>
-    </entry>
    <entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-common/2.6.0-cdh5.7.0/hadoop-common-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/fs/FileSystem.class">
      <provider selected="true" editor-type-id="text-editor">
        <state relative-caret-position="455">
@@ -943,16 +944,6 @@
        </state>
      </provider>
    </entry>
-    <entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="161">
-          <caret line="19" column="25" lean-forward="false" selection-start-line="19" selection-start-column="25" selection-end-line="19" selection-end-column="25" />
-          <folding>
-            <element signature="imports" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
    <entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Mapper.class">
      <provider selected="true" editor-type-id="text-editor">
        <state relative-caret-position="184">
@@ -991,10 +982,28 @@
        </state>
      </provider>
    </entry>
+    <entry file="file://$PROJECT_DIR$/pom.xml">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="138">
+          <caret line="6" column="14" lean-forward="false" selection-start-line="6" selection-start-column="14" selection-end-line="6" selection-end-column="14" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="23">
+          <caret line="19" column="25" lean-forward="false" selection-start-line="19" selection-start-column="25" selection-end-line="19" selection-end-column="25" />
+          <folding>
+            <element signature="imports" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
    <entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java">
      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="437">
-          <caret line="95" column="68" lean-forward="false" selection-start-line="95" selection-start-column="68" selection-end-line="95" selection-end-column="68" />
+        <state relative-caret-position="755">
+          <caret line="135" column="0" lean-forward="true" selection-start-line="135" selection-start-column="0" selection-end-line="135" selection-end-column="0" />
          <folding>
            <element signature="imports" expanded="true" />
          </folding>
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@

  <groupId>com.fjy.hadoop</groupId>
  <artifactId>hadoopstudy</artifactId>
-  <version>1.0-SNAPSHOT</version>
+  <version>0.3</version>
  <packaging>jar</packaging>

  <name>hadoopstudy</name>
--- a/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java
+++ b/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java
@@ -119,6 +119,10 @@ public class WordCountApp {
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);

+        //通过job设置combiner处理类，逻辑上与reduce一致，注意，如果要计算平均数等不能使用Combiner!
+        job.setCombinerClass(MyReducer.class);
+
+
        //设置作业处理输出结果的输出路径
        FileOutputFormat.setOutputPath(job, new Path(args[1]));