test/hadoopstudy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

使用Partitioner

Browse Source
This commit is contained in:
F嘉阳
2018-04-20 16:40:51 +08:00
parent c272dbec7e
commit eba6a66132
4 changed files with 506 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
.idea/uiDesigner.xml generated Normal file
View File

@@ -0,0 +1,124 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Palette2">
<group name="Swing">
<item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
</item>
<item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
</item>
<item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.png" removable="false" auto-create-binding="false" can-attach-label="true">
<default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
</item>
<item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
<initial-values>
<property name="text" value="Button" />
</initial-values>
</item>
<item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="RadioButton" />
</initial-values>
</item>
<item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="CheckBox" />
</initial-values>
</item>
<item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
<initial-values>
<property name="text" value="Label" />
</initial-values>
</item>
<item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
</item>
<item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
</item>
<item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
<preferred-size width="-1" height="20" />
</default-constraints>
</item>
<item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
</item>
<item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
</item>
</group>
</component>
</project>

315
.idea/workspace.xml generated
View File

@@ -2,9 +2,12 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="8ccbe0a5-cdf8-4091-a84b-916013277899" name="Default" comment="">
<change beforePath="" afterPath="$PROJECT_DIR$/.idea/libraries/Maven__com_kumkee_UserAgentParser_0_0_1.xml" />
<change beforePath="" afterPath="$PROJECT_DIR$/.idea/uiDesigner.xml" />
<change beforePath="" afterPath="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountPartitionerApp.java" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change beforePath="$PROJECT_DIR$/hadoopstudy.iml" afterPath="$PROJECT_DIR$/hadoopstudy.iml" />
<change beforePath="$PROJECT_DIR$/pom.xml" afterPath="$PROJECT_DIR$/pom.xml" />
<change beforePath="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java" afterPath="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java" />
</list>
<ignored path="$PROJECT_DIR$/out/" />
<ignored path="$PROJECT_DIR$/target/" />
@@ -17,11 +20,11 @@
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="pom.xml" pinned="false" current-in-tab="false">
<file leaf-file-name="pom.xml" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="138">
<caret line="6" column="14" lean-forward="false" selection-start-line="6" selection-start-column="14" selection-end-line="6" selection-end-column="14" />
<state relative-caret-position="203">
<caret line="30" column="17" lean-forward="false" selection-start-line="30" selection-start-column="17" selection-end-line="30" selection-end-column="17" />
<folding />
</state>
</provider>
@@ -30,8 +33,8 @@
<file leaf-file-name="HDFSApp.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="23">
<caret line="19" column="25" lean-forward="false" selection-start-line="19" selection-start-column="25" selection-end-line="19" selection-end-column="25" />
<state relative-caret-position="207">
<caret line="21" column="15" lean-forward="false" selection-start-line="21" selection-start-column="15" selection-end-line="21" selection-end-column="15" />
<folding>
<element signature="imports" expanded="true" />
</folding>
@@ -39,11 +42,11 @@
</provider>
</entry>
</file>
<file leaf-file-name="WordCountApp.java" pinned="false" current-in-tab="true">
<file leaf-file-name="WordCountApp.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="755">
<caret line="135" column="0" lean-forward="true" selection-start-line="135" selection-start-column="0" selection-end-line="135" selection-end-column="0" />
<caret line="135" column="0" lean-forward="false" selection-start-line="135" selection-start-column="0" selection-end-line="135" selection-end-column="0" />
<folding>
<element signature="imports" expanded="true" />
</folding>
@@ -51,6 +54,16 @@
</provider>
</entry>
</file>
<file leaf-file-name="WordCountPartitionerApp.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountPartitionerApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="552">
<caret line="150" column="0" lean-forward="false" selection-start-line="150" selection-start-column="0" selection-end-line="150" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="Reducer.class" pinned="false" current-in-tab="false">
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Reducer.class">
<provider selected="true" editor-type-id="text-editor">
@@ -124,8 +137,11 @@
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java" />
<option value="$PROJECT_DIR$/pom.xml" />
<option value="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java" />
<option value="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountPartitionerApp.java" />
<option value="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgent.java" />
<option value="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java" />
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
@@ -171,10 +187,9 @@
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="AndroidView" />
<pane id="Scope" />
<pane id="PackagesPane" />
<pane id="Scratches" />
<pane id="AndroidView" />
<pane id="ProjectPane">
<subPane>
<expand>
@@ -237,6 +252,7 @@
<select />
</subPane>
</pane>
<pane id="Scratches" />
</panes>
</component>
<component name="PropertiesComponent">
@@ -246,6 +262,11 @@
<property name="settings.editor.selected.configurable" value="fileTemplates" />
<property name="last_opened_file_path" value="$PROJECT_DIR$/../第二章代码" />
</component>
<component name="RecentsManager">
<key name="CopyClassDialog.RECENTS_KEY">
<recent name="com.fjy.hadoop.mapreduce" />
</key>
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
@@ -258,7 +279,7 @@
</list>
</option>
</component>
<component name="RunManager" selected="JUnit.HDFSApp.delete">
<component name="RunManager" selected="Application.UserAgentTest">
<configuration default="true" type="Applet" factoryName="Applet">
<option name="HTML_USED" value="false" />
<option name="WIDTH" value="400" />
@@ -266,6 +287,25 @@
<option name="POLICY_FILE" value="$APPLICATION_HOME_DIR$/bin/appletviewer.policy" />
<module />
</configuration>
<configuration name="UserAgentTest" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea">
<pattern>
<option name="PATTERN" value="com.fjy.hadoop.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
<option name="MAIN_CLASS_NAME" value="com.fjy.hadoop.UserAgentTest" />
<option name="VM_PARAMETERS" />
<option name="PROGRAM_PARAMETERS" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="ENABLE_SWING_INSPECTOR" value="false" />
<option name="ENV_VARIABLES" />
<option name="PASS_PARENT_ENVS" value="true" />
<module name="hadoopstudy" />
<envs />
</configuration>
<configuration default="true" type="Application" factoryName="Application">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<option name="MAIN_CLASS_NAME" />
@@ -280,31 +320,6 @@
<module name="" />
<envs />
</configuration>
<configuration name="HDFSApp.copyFromLocalFile" type="JUnit" factoryName="JUnit" temporary="true" nameIsGenerated="true">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea">
<pattern>
<option name="PATTERN" value="com.fjy.hadoop.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
<module name="hadoopstudy" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="PACKAGE_NAME" value="com.fjy.hadoop" />
<option name="MAIN_CLASS_NAME" value="com.fjy.hadoop.HDFSApp" />
<option name="METHOD_NAME" value="copyFromLocalFile" />
<option name="TEST_OBJECT" value="method" />
<option name="VM_PARAMETERS" value="-ea" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" value="%MODULE_WORKING_DIR%" />
<option name="ENV_VARIABLES" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<envs />
<patterns />
</configuration>
<configuration name="HDFSApp.copyFromLocalFileWithProgress" type="JUnit" factoryName="JUnit" temporary="true" nameIsGenerated="true">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea">
<pattern>
@@ -466,19 +481,19 @@
<predefined_log_file id="idea.log" enabled="true" />
</configuration>
<list size="5">
<item index="0" class="java.lang.String" itemvalue="JUnit.HDFSApp.copyFromLocalFile" />
<item index="1" class="java.lang.String" itemvalue="JUnit.HDFSApp.copyFromLocalFileWithProgress" />
<item index="2" class="java.lang.String" itemvalue="JUnit.HDFSApp.copyToLocalFile" />
<item index="3" class="java.lang.String" itemvalue="JUnit.HDFSApp.listFile" />
<item index="4" class="java.lang.String" itemvalue="JUnit.HDFSApp.delete" />
<item index="0" class="java.lang.String" itemvalue="JUnit.HDFSApp.copyFromLocalFileWithProgress" />
<item index="1" class="java.lang.String" itemvalue="JUnit.HDFSApp.copyToLocalFile" />
<item index="2" class="java.lang.String" itemvalue="JUnit.HDFSApp.listFile" />
<item index="3" class="java.lang.String" itemvalue="JUnit.HDFSApp.delete" />
<item index="4" class="java.lang.String" itemvalue="Application.UserAgentTest" />
</list>
<recent_temporary>
<list size="5">
<item index="0" class="java.lang.String" itemvalue="JUnit.HDFSApp.delete" />
<item index="1" class="java.lang.String" itemvalue="JUnit.HDFSApp.listFile" />
<item index="2" class="java.lang.String" itemvalue="JUnit.HDFSApp.copyToLocalFile" />
<item index="3" class="java.lang.String" itemvalue="JUnit.HDFSApp.copyFromLocalFileWithProgress" />
<item index="4" class="java.lang.String" itemvalue="JUnit.HDFSApp.copyFromLocalFile" />
<item index="0" class="java.lang.String" itemvalue="Application.UserAgentTest" />
<item index="1" class="java.lang.String" itemvalue="JUnit.HDFSApp.delete" />
<item index="2" class="java.lang.String" itemvalue="JUnit.HDFSApp.listFile" />
<item index="3" class="java.lang.String" itemvalue="JUnit.HDFSApp.copyToLocalFile" />
<item index="4" class="java.lang.String" itemvalue="JUnit.HDFSApp.copyFromLocalFileWithProgress" />
</list>
</recent_temporary>
</component>
@@ -506,7 +521,9 @@
<workItem from="1523322335998" duration="659000" />
<workItem from="1523606327796" duration="135000" />
<workItem from="1523931905005" duration="6215000" />
<workItem from="1524056118416" duration="3187000" />
<workItem from="1524056118416" duration="4576000" />
<workItem from="1524146174789" duration="602000" />
<workItem from="1524196421728" duration="3568000" />
</task>
<task id="LOCAL-00001" summary="建立初始工程">
<created>1521904777876</created>
@@ -550,7 +567,14 @@
<option name="project" value="LOCAL" />
<updated>1524058535951</updated>
</task>
<option name="localTasksCounter" value="7" />
<task id="LOCAL-00007" summary="使用Combiner加快处理效率">
<created>1524061191596</created>
<option name="number" value="00007" />
<option name="presentableId" value="LOCAL-00007" />
<option name="project" value="LOCAL" />
<updated>1524061191596</updated>
</task>
<option name="localTasksCounter" value="8" />
<servers />
</component>
<component name="TestHistory">
@@ -586,7 +610,7 @@
</history-entry>
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="30194000" />
<option name="totallyTimeSpent" value="35753000" />
</component>
<component name="ToolWindowManager">
<frame x="-8" y="-8" width="1936" height="1056" extended-state="6" />
@@ -594,30 +618,30 @@
<layout>
<window_info id="Palette" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Messages" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32936078" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Palette&#9;" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Image Layers" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Java Enterprise" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="8" side_tool="false" content_ui="tabs" />
<window_info id="Capture Analysis" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32936078" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.32936078" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.17803837" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.1684435" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="UI Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Theme Preview" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.38159254" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.38159254" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
<window_info id="Messages" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32936078" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32936078" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
</layout>
@@ -634,13 +658,88 @@
<MESSAGE value="单节点MapReduce开发WordCount" />
<MESSAGE value="补全注释" />
<MESSAGE value="工作前自动删除已有文件" />
<option name="LAST_COMMIT_MESSAGE" value="工作前自动删除已有文件" />
<MESSAGE value="使用Combiner加快处理效率" />
<option name="LAST_COMMIT_MESSAGE" value="使用Combiner加快处理效率" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager />
<watches-manager />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="138">
<caret line="6" column="14" lean-forward="false" selection-start-line="6" selection-start-column="14" selection-end-line="6" selection-end-column="14" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="437">
<caret line="19" column="25" lean-forward="false" selection-start-line="19" selection-start-column="25" selection-end-line="19" selection-end-column="25" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Reducer.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="207">
<caret line="17" column="13" lean-forward="false" selection-start-line="17" selection-start-column="13" selection-end-line="17" selection-end-column="13" />
<folding>
<marker date="1521904733592" expanded="false" signature="0:104" ph="/.../" />
<marker date="1521904733592" expanded="false" signature="151:462" ph="..." />
<marker date="1521904733592" expanded="true" signature="464:495" ph="@{...}" />
<marker date="1521904733592" expanded="true" signature="574:581" ph="{}" />
<marker date="1521904733592" expanded="true" signature="708:715" ph="{}" />
<marker date="1521904733592" expanded="true" signature="880:1048" ph="{...}" />
<marker date="1521904733592" expanded="true" signature="1177:1184" ph="{}" />
<marker date="1521904733592" expanded="true" signature="1306:1766" ph="{...}" />
<marker date="1521904733592" expanded="true" signature="1861:1905" ph="{...}" />
<marker date="1521904733592" expanded="true" signature="1888:1899" ph="{}" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Mapper.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="184">
<caret line="13" column="13" lean-forward="false" selection-start-line="13" selection-start-column="13" selection-end-line="13" selection-end-column="13" />
<folding>
<marker date="1521904733592" expanded="false" signature="0:104" ph="/.../" />
<marker date="1521904733592" expanded="false" signature="151:304" ph="..." />
<marker date="1521904733592" expanded="true" signature="306:321" ph="@{...}" />
<marker date="1521904733592" expanded="true" signature="398:405" ph="{}" />
<marker date="1521904733592" expanded="true" signature="531:538" ph="{}" />
<marker date="1521904733592" expanded="true" signature="688:730" ph="{...}" />
<marker date="1521904733592" expanded="true" signature="858:865" ph="{}" />
<marker date="1521904733592" expanded="true" signature="986:1248" ph="{...}" />
<marker date="1521904733592" expanded="true" signature="1340:1384" ph="{...}" />
<marker date="1521904733592" expanded="true" signature="1367:1378" ph="{}" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/AppTest.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="115">
<caret line="7" column="28" lean-forward="false" selection-start-line="7" selection-start-column="28" selection-end-line="7" selection-end-column="28" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="2852">
<caret line="135" column="0" lean-forward="true" selection-start-line="135" selection-start-column="0" selection-end-line="135" selection-end-column="0" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="782">
@@ -936,11 +1035,22 @@
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/App.java" />
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/Test.java" />
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/AppTest.java">
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Reducer.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="115">
<caret line="7" column="28" lean-forward="false" selection-start-line="7" selection-start-column="28" selection-end-line="7" selection-end-column="28" />
<folding />
<state relative-caret-position="207">
<caret line="17" column="13" lean-forward="false" selection-start-line="17" selection-start-column="13" selection-end-line="17" selection-end-column="13" />
<folding>
<marker date="1521904733592" expanded="false" signature="0:104" ph="/.../" />
<marker date="1521904733592" expanded="false" signature="151:462" ph="..." />
<marker date="1521904733592" expanded="true" signature="464:495" ph="@{...}" />
<marker date="1521904733592" expanded="true" signature="574:581" ph="{}" />
<marker date="1521904733592" expanded="true" signature="708:715" ph="{}" />
<marker date="1521904733592" expanded="true" signature="880:1048" ph="{...}" />
<marker date="1521904733592" expanded="true" signature="1177:1184" ph="{}" />
<marker date="1521904733592" expanded="true" signature="1306:1766" ph="{...}" />
<marker date="1521904733592" expanded="true" signature="1861:1905" ph="{...}" />
<marker date="1521904733592" expanded="true" signature="1888:1899" ph="{}" />
</folding>
</state>
</provider>
</entry>
@@ -963,53 +1073,74 @@
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Reducer.class">
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/AppTest.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="207">
<caret line="17" column="13" lean-forward="false" selection-start-line="17" selection-start-column="13" selection-end-line="17" selection-end-column="13" />
<folding>
<marker date="1521904733592" expanded="false" signature="0:104" ph="/.../" />
<marker date="1521904733592" expanded="false" signature="151:462" ph="..." />
<marker date="1521904733592" expanded="true" signature="464:495" ph="@{...}" />
<marker date="1521904733592" expanded="true" signature="574:581" ph="{}" />
<marker date="1521904733592" expanded="true" signature="708:715" ph="{}" />
<marker date="1521904733592" expanded="true" signature="880:1048" ph="{...}" />
<marker date="1521904733592" expanded="true" signature="1177:1184" ph="{}" />
<marker date="1521904733592" expanded="true" signature="1306:1766" ph="{...}" />
<marker date="1521904733592" expanded="true" signature="1861:1905" ph="{...}" />
<marker date="1521904733592" expanded="true" signature="1888:1899" ph="{}" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="138">
<caret line="6" column="14" lean-forward="false" selection-start-line="6" selection-start-column="14" selection-end-line="6" selection-end-column="14" />
<state relative-caret-position="115">
<caret line="7" column="28" lean-forward="false" selection-start-line="7" selection-start-column="28" selection-end-line="7" selection-end-column="28" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="23">
<caret line="19" column="25" lean-forward="false" selection-start-line="19" selection-start-column="25" selection-end-line="19" selection-end-column="25" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="755">
<caret line="135" column="0" lean-forward="true" selection-start-line="135" selection-start-column="0" selection-end-line="135" selection-end-column="0" />
<caret line="135" column="0" lean-forward="false" selection-start-line="135" selection-start-column="0" selection-end-line="135" selection-end-column="0" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="207">
<caret line="21" column="15" lean-forward="false" selection-start-line="21" selection-start-column="15" selection-end-line="21" selection-end-column="15" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Partitioner.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="184">
<caret line="11" column="7" lean-forward="true" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<folding />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Job.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="298">
<caret line="496" column="16" lean-forward="false" selection-start-line="496" selection-start-column="16" selection-end-line="496" selection-end-column="16" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountPartitionerApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="552">
<caret line="150" column="0" lean-forward="false" selection-start-line="150" selection-start-column="0" selection-end-line="150" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="203">
<caret line="30" column="17" lean-forward="false" selection-start-line="30" selection-start-column="17" selection-end-line="30" selection-end-column="17" />
<folding />
</state>
</provider>
</entry>
</component>
<component name="masterDetails">
<states>

2
pom.xml
View File

@@ -4,7 +4,7 @@
<groupId>com.fjy.hadoop</groupId>
<artifactId>hadoopstudy</artifactId>
<version>0.3</version>
<version>0.4</version>
<packaging>jar</packaging>
<name>hadoopstudy</name>

158
src/main/java/com/fjy/hadoop/mapreduce/WordCountPartitionerApp.java Normal file
View File

@@ -0,0 +1,158 @@
package com.fjy.hadoop.mapreduce;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
/**
* 使用MapReduce开发Partitioner组件应用
* @author F嘉阳
* @date 2018-04-20
*/
public class WordCountPartitionerApp {
/**
* Map：读取输入文件
* Text:类似字符串
*/
public static class MyMapper extends Mapper<LongWritable, Text, Text, LongWritable> {
LongWritable one = new LongWritable(1);
/**
* @param key 偏移量
* @param value 每行的字符串
* @param context 上下文
* @throws IOException
* @throws InterruptedException
*/
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
/*super.map(key, value, context);*/
//接收到每一行数据
String line = value.toString();
String[] words = line.split(" ");//以空格分隔
//每一个空格是一个手机品牌，另一个是销售数量
context.write(new Text(words[0]), new LongWritable(Long.parseLong(words[1])));
}
}
/**
* Reduce 归并操作
* LongWritable：文本出现的次数/求和后的次数
*/
public static class MyReducer extends Reducer<Text, LongWritable, Text, LongWritable> {
/**
* @param key
* @param values 相同偏移量的集合
* @param context
* @throws IOException
* @throws InterruptedException
*/
@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
//super.reduce(key, values, context);
long sum = 0;
for (LongWritable value : values) {
//求key出现的次数和总和
sum += value.get();
}
//统计结果的输出
context.write(key, new LongWritable(sum));
}
}
/**
* Partitioner处理类
*/
public static class MyPartitioner extends Partitioner<Text,LongWritable>{
@Override
public int getPartition(Text key, LongWritable value, int i) {
if ("xiaomi".equals(key.toString())) {
return 0;//若为xiaomi则交由0 ReduceTask处理
}
if ("huawei".equals(key.toString())) {
return 1;//若为huawei则交由1 ReduceTask处理
}
if ("iphone".equals(key.toString())) {
return 2;//若为iphone则交由2 ReduceTask处理
}
return 3;//若为其他，则交由3 ReduceTask处理
}
}
/**
* 定义Driver,封装MapReduce作业的所有信息
*/
public static void main(String[] args) throws Exception {
//创建Configuration
Configuration configuration = new Configuration();
//清理已存在的输出目录
Path outputPath = new Path(args[1]);
FileSystem fileSystem = FileSystem.get(configuration);
if (fileSystem.exists(outputPath)){
fileSystem.delete(outputPath);
System.out.println("The exist files had been deleted!");
}
//创建作业
Job job = Job.getInstance(configuration, "wordcount");
//设置作业的主类
job.setJarByClass(WordCountPartitionerApp.class);
//作业处理的输入路径
FileInputFormat.setInputPaths(job, new Path(args[0]));
//设置map相关参数
job.setMapperClass(MyMapper.class);
//设置map输出的key的类型
job.setMapOutputKeyClass(Text.class);
//设置map输出的value的类型
job.setMapOutputValueClass(LongWritable.class);
//设置Reduce相关参数
job.setReducerClass(MyReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
//设置job的Partition
job.setPartitionerClass(MyPartitioner.class);
//设置四个reducer，每个分区一个，否则Partitioner配置不生效
job.setNumReduceTasks(4);
//设置作业处理输出结果的输出路径
FileOutputFormat.setOutputPath(job, new Path(args[1]));
//作业提交
job.waitForCompletion(true);
//作业完成后退出
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}