test/hadoopstudy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

MapReduce日志处理

Browse Source
This commit is contained in:
F嘉阳
2018-04-20 23:16:44 +08:00
parent 4623552076
commit 0301d1d924
4 changed files with 419 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

229
.idea/workspace.xml generated
View File

@@ -2,8 +2,10 @@
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="8ccbe0a5-cdf8-4091-a84b-916013277899" name="Default" comment=""> <list default="true" id="8ccbe0a5-cdf8-4091-a84b-916013277899" name="Default" comment="">
<change beforePath="" afterPath="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/hadoop/project/LogApp.java" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" /> <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change beforePath="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java" afterPath="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java" /> <change beforePath="$PROJECT_DIR$/hadoopstudy.iml" afterPath="$PROJECT_DIR$/hadoopstudy.iml" />
<change beforePath="$PROJECT_DIR$/pom.xml" afterPath="$PROJECT_DIR$/pom.xml" />
</list> </list>
<ignored path="$PROJECT_DIR$/out/" /> <ignored path="$PROJECT_DIR$/out/" />
<ignored path="$PROJECT_DIR$/target/" /> <ignored path="$PROJECT_DIR$/target/" />
@@ -19,8 +21,8 @@
<file leaf-file-name="pom.xml" pinned="false" current-in-tab="false"> <file leaf-file-name="pom.xml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/pom.xml"> <entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="410"> <state relative-caret-position="138">
<caret line="40" column="43" lean-forward="false" selection-start-line="40" selection-start-column="43" selection-end-line="40" selection-end-column="43" /> <caret line="6" column="16" lean-forward="false" selection-start-line="6" selection-start-column="16" selection-end-line="6" selection-end-column="16" />
<folding /> <folding />
</state> </state>
</provider> </provider>
@@ -29,11 +31,22 @@
<file leaf-file-name="UserAgentTest.java" pinned="false" current-in-tab="true"> <file leaf-file-name="UserAgentTest.java" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java"> <entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-629"> <state relative-caret-position="364">
<caret line="23" column="23" lean-forward="true" selection-start-line="23" selection-start-column="23" selection-end-line="23" selection-end-column="23" /> <caret line="23" column="20" lean-forward="false" selection-start-line="23" selection-start-column="20" selection-end-line="23" selection-end-column="20" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="LogApp.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/hadoop/project/LogApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="713">
<caret line="55" column="34" lean-forward="true" selection-start-line="55" selection-start-column="34" selection-end-line="55" selection-end-column="34" />
<folding> <folding>
<element signature="imports" expanded="true" /> <element signature="imports" expanded="true" />
<element signature="e#790#807#0" expanded="true" />
</folding> </folding>
</state> </state>
</provider> </provider>
@@ -42,7 +55,7 @@
<file leaf-file-name="HDFSApp.java" pinned="false" current-in-tab="false"> <file leaf-file-name="HDFSApp.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java"> <entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="207"> <state relative-caret-position="-2511">
<caret line="21" column="15" lean-forward="false" selection-start-line="21" selection-start-column="15" selection-end-line="21" selection-end-column="15" /> <caret line="21" column="15" lean-forward="false" selection-start-line="21" selection-start-column="15" selection-end-line="21" selection-end-column="15" />
<folding> <folding>
<element signature="imports" expanded="true" /> <element signature="imports" expanded="true" />
@@ -54,8 +67,8 @@
<file leaf-file-name="WordCountApp.java" pinned="false" current-in-tab="false"> <file leaf-file-name="WordCountApp.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java"> <entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="755"> <state relative-caret-position="617">
<caret line="135" column="0" lean-forward="false" selection-start-line="135" selection-start-column="0" selection-end-line="135" selection-end-column="0" /> <caret line="67" column="36" lean-forward="false" selection-start-line="67" selection-start-column="36" selection-end-line="67" selection-end-column="36" />
<folding> <folding>
<element signature="imports" expanded="true" /> <element signature="imports" expanded="true" />
</folding> </folding>
@@ -63,16 +76,6 @@
</provider> </provider>
</entry> </entry>
</file> </file>
<file leaf-file-name="WordCountPartitionerApp.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountPartitionerApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="552">
<caret line="150" column="0" lean-forward="false" selection-start-line="150" selection-start-column="0" selection-end-line="150" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="Reducer.class" pinned="false" current-in-tab="false"> <file leaf-file-name="Reducer.class" pinned="false" current-in-tab="false">
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Reducer.class"> <entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Reducer.class">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
@@ -149,8 +152,9 @@
<option value="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java" /> <option value="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java" />
<option value="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountPartitionerApp.java" /> <option value="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountPartitionerApp.java" />
<option value="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgent.java" /> <option value="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgent.java" />
<option value="$PROJECT_DIR$/pom.xml" />
<option value="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java" /> <option value="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java" />
<option value="$PROJECT_DIR$/pom.xml" />
<option value="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/hadoop/project/LogApp.java" />
</list> </list>
</option> </option>
</component> </component>
@@ -196,9 +200,8 @@
<foldersAlwaysOnTop value="true" /> <foldersAlwaysOnTop value="true" />
</navigator> </navigator>
<panes> <panes>
<pane id="Scratches" />
<pane id="AndroidView" /> <pane id="AndroidView" />
<pane id="Scope" />
<pane id="PackagesPane" />
<pane id="ProjectPane"> <pane id="ProjectPane">
<subPane> <subPane>
<expand> <expand>
@@ -230,6 +233,15 @@
<item name="src" type="462c0819:PsiDirectoryNode" /> <item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" /> <item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" /> <item name="java" type="462c0819:PsiDirectoryNode" />
<item name="hadoop" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hadoopstudy" type="b2602c69:ProjectViewProjectNode" />
<item name="hadoopstudy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="hadoop" type="462c0819:PsiDirectoryNode" />
<item name="mapreduce" type="462c0819:PsiDirectoryNode" /> <item name="mapreduce" type="462c0819:PsiDirectoryNode" />
</path> </path>
<path> <path>
@@ -261,7 +273,8 @@
<select /> <select />
</subPane> </subPane>
</pane> </pane>
<pane id="Scratches" /> <pane id="Scope" />
<pane id="PackagesPane" />
</panes> </panes>
</component> </component>
<component name="PropertiesComponent"> <component name="PropertiesComponent">
@@ -532,7 +545,8 @@
<workItem from="1523931905005" duration="6215000" /> <workItem from="1523931905005" duration="6215000" />
<workItem from="1524056118416" duration="4576000" /> <workItem from="1524056118416" duration="4576000" />
<workItem from="1524146174789" duration="602000" /> <workItem from="1524146174789" duration="602000" />
<workItem from="1524196421728" duration="4974000" /> <workItem from="1524196421728" duration="5000000" />
<workItem from="1524232464914" duration="3275000" />
</task> </task>
<task id="LOCAL-00001" summary="建立初始工程"> <task id="LOCAL-00001" summary="建立初始工程">
<created>1521904777876</created> <created>1521904777876</created>
@@ -597,13 +611,17 @@
<option name="project" value="LOCAL" /> <option name="project" value="LOCAL" />
<updated>1524214734524</updated> <updated>1524214734524</updated>
</task> </task>
<option name="localTasksCounter" value="10" /> <task id="LOCAL-00010" summary="单机版日志处理完成">
<created>1524215062883</created>
<option name="number" value="00010" />
<option name="presentableId" value="LOCAL-00010" />
<option name="project" value="LOCAL" />
<updated>1524215062883</updated>
</task>
<option name="localTasksCounter" value="11" />
<servers /> <servers />
</component> </component>
<component name="TestHistory"> <component name="TestHistory">
<history-entry file="HDFSApp_copyToLocalFile - 2018.04.08 at 12h 00m 17s.xml">
<configuration name="HDFSApp.copyToLocalFile" configurationId="JUnit" />
</history-entry>
<history-entry file="HDFSApp_copyToLocalFile - 2018.04.08 at 12h 03m 40s.xml"> <history-entry file="HDFSApp_copyToLocalFile - 2018.04.08 at 12h 03m 40s.xml">
<configuration name="HDFSApp.copyToLocalFile" configurationId="JUnit" /> <configuration name="HDFSApp.copyToLocalFile" configurationId="JUnit" />
</history-entry> </history-entry>
@@ -631,9 +649,12 @@
<history-entry file="UserAgentTest_testReadFile - 2018.04.20 at 17h 03m 15s.xml"> <history-entry file="UserAgentTest_testReadFile - 2018.04.20 at 17h 03m 15s.xml">
<configuration name="UserAgentTest.testReadFile" configurationId="JUnit" /> <configuration name="UserAgentTest.testReadFile" configurationId="JUnit" />
</history-entry> </history-entry>
<history-entry file="UserAgentTest_testReadFile - 2018.04.20 at 23h 15m 30s.xml">
<configuration name="UserAgentTest.testReadFile" configurationId="JUnit" />
</history-entry>
</component> </component>
<component name="TimeTrackingManager"> <component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="37159000" /> <option name="totallyTimeSpent" value="40460000" />
</component> </component>
<component name="ToolWindowManager"> <component name="ToolWindowManager">
<frame x="-8" y="-8" width="1936" height="1056" extended-state="6" /> <frame x="-8" y="-8" width="1936" height="1056" extended-state="6" />
@@ -648,19 +669,19 @@
<window_info id="Capture Analysis" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> <window_info id="Capture Analysis" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" /> <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> <window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.31310943" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" /> <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="true" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.31310943" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" /> <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" /> <window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" /> <window_info id="Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.1684435" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" /> <window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.17004265" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> <window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> <window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="UI Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" /> <window_info id="UI Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Theme Preview" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> <window_info id="Theme Preview" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.38159254" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.38159254" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
<window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" /> <window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" /> <window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" /> <window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
@@ -684,31 +705,14 @@
<MESSAGE value="使用Combiner加快处理效率" /> <MESSAGE value="使用Combiner加快处理效率" />
<MESSAGE value="使用Partitioner" /> <MESSAGE value="使用Partitioner" />
<MESSAGE value="处理本地日志记录" /> <MESSAGE value="处理本地日志记录" />
<option name="LAST_COMMIT_MESSAGE" value="处理本地日志记录" /> <MESSAGE value="单机版日志处理完成" />
<option name="LAST_COMMIT_MESSAGE" value="单机版日志处理完成" />
</component> </component>
<component name="XDebuggerManager"> <component name="XDebuggerManager">
<breakpoint-manager /> <breakpoint-manager />
<watches-manager /> <watches-manager />
</component> </component>
<component name="editorHistoryManager"> <component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="138">
<caret line="6" column="14" lean-forward="false" selection-start-line="6" selection-start-column="14" selection-end-line="6" selection-end-column="14" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="437">
<caret line="19" column="25" lean-forward="false" selection-start-line="19" selection-start-column="25" selection-end-line="19" selection-end-column="25" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Reducer.class"> <entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Reducer.class">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="207"> <state relative-caret-position="207">
@@ -1098,39 +1102,10 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/AppTest.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="115">
<caret line="7" column="28" lean-forward="false" selection-start-line="7" selection-start-column="28" selection-end-line="7" selection-end-column="28" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="755">
<caret line="135" column="0" lean-forward="false" selection-start-line="135" selection-start-column="0" selection-end-line="135" selection-end-column="0" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="207">
<caret line="21" column="15" lean-forward="false" selection-start-line="21" selection-start-column="15" selection-end-line="21" selection-end-column="15" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Partitioner.class"> <entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Partitioner.class">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="184"> <state relative-caret-position="184">
<caret line="11" column="7" lean-forward="true" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" /> <caret line="11" column="7" lean-forward="true" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<folding />
</state> </state>
</provider> </provider>
</entry> </entry>
@@ -1138,23 +1113,6 @@
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="298"> <state relative-caret-position="298">
<caret line="496" column="16" lean-forward="false" selection-start-line="496" selection-start-column="16" selection-end-line="496" selection-end-column="16" /> <caret line="496" column="16" lean-forward="false" selection-start-line="496" selection-start-column="16" selection-end-line="496" selection-end-column="16" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountPartitionerApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="552">
<caret line="150" column="0" lean-forward="false" selection-start-line="150" selection-start-column="0" selection-end-line="150" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="410">
<caret line="40" column="43" lean-forward="false" selection-start-line="40" selection-start-column="43" selection-end-line="40" selection-end-column="43" />
<folding />
</state> </state>
</provider> </provider>
</entry> </entry>
@@ -1166,13 +1124,82 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java"> <entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountPartitionerApp.java">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-629"> <state relative-caret-position="276">
<caret line="23" column="23" lean-forward="true" selection-start-line="23" selection-start-column="23" selection-end-line="23" selection-end-column="23" /> <caret line="75" column="26" lean-forward="true" selection-start-line="75" selection-start-column="26" selection-end-line="75" selection-end-column="26" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="617">
<caret line="67" column="36" lean-forward="false" selection-start-line="67" selection-start-column="36" selection-end-line="67" selection-end-column="36" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-common/2.6.0-cdh5.7.0/hadoop-common-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/io/Text.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="211">
<caret line="53" column="11" lean-forward="false" selection-start-line="53" selection-start-column="11" selection-end-line="53" selection-end-column="11" />
<folding />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/TaskInputOutputContext.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="345">
<caret line="20" column="9" lean-forward="false" selection-start-line="20" selection-start-column="9" selection-end-line="20" selection-end-column="9" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="138">
<caret line="6" column="16" lean-forward="false" selection-start-line="6" selection-start-column="16" selection-end-line="6" selection-end-column="16" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-2511">
<caret line="21" column="15" lean-forward="false" selection-start-line="21" selection-start-column="15" selection-end-line="21" selection-end-column="15" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/hadoop/project/LogApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="713">
<caret line="55" column="34" lean-forward="true" selection-start-line="55" selection-start-column="34" selection-end-line="55" selection-end-column="34" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/AppTest.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="115">
<caret line="7" column="28" lean-forward="false" selection-start-line="7" selection-start-column="28" selection-end-line="7" selection-end-column="28" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="364">
<caret line="23" column="20" lean-forward="false" selection-start-line="23" selection-start-column="20" selection-end-line="23" selection-end-column="20" />
<folding> <folding>
<element signature="imports" expanded="true" /> <element signature="imports" expanded="true" />
<element signature="e#790#807#0" expanded="true" />
</folding> </folding>
</state> </state>
</provider> </provider>

148
hadoopstudy.iml
View File

@@ -13,79 +13,79 @@
<orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.10" level="project" /> <orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.10" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.1" level="project" /> <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.1" level="project" />
<orderEntry type="library" name="Maven: com.kumkee:UserAgentParser:0.0.1" level="project" /> <orderEntry type="library" name="Maven: com.kumkee:UserAgentParser:0.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-client:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-client:2.6.0-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-common:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-common:2.6.0-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:11.0.2" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.guava:guava:11.0.2" level="project" />
<orderEntry type="library" name="Maven: commons-cli:commons-cli:1.2" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: commons-cli:commons-cli:1.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.1.1" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.commons:commons-math3:3.1.1" level="project" />
<orderEntry type="library" name="Maven: xmlenc:xmlenc:0.52" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: xmlenc:xmlenc:0.52" level="project" />
<orderEntry type="library" name="Maven: commons-httpclient:commons-httpclient:3.1" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: commons-httpclient:commons-httpclient:3.1" level="project" />
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.4" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: commons-codec:commons-codec:1.4" level="project" />
<orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: commons-io:commons-io:2.4" level="project" />
<orderEntry type="library" name="Maven: commons-net:commons-net:3.1" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: commons-net:commons-net:3.1" level="project" />
<orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.2" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: commons-collections:commons-collections:3.2.2" level="project" />
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.3" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: commons-logging:commons-logging:1.1.3" level="project" />
<orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: log4j:log4j:1.2.17" level="project" />
<orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: commons-lang:commons-lang:2.6" level="project" />
<orderEntry type="library" name="Maven: commons-configuration:commons-configuration:1.6" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: commons-configuration:commons-configuration:1.6" level="project" />
<orderEntry type="library" name="Maven: commons-digester:commons-digester:1.8" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: commons-digester:commons-digester:1.8" level="project" />
<orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils:1.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: commons-beanutils:commons-beanutils:1.7.0" level="project" />
<orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils-core:1.8.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: commons-beanutils:commons-beanutils-core:1.8.0" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.5" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.slf4j:slf4j-api:1.7.5" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.5" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.slf4j:slf4j-log4j12:1.7.5" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.8.8" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.codehaus.jackson:jackson-core-asl:1.8.8" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.8.8" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.8.8" level="project" />
<orderEntry type="library" name="Maven: org.apache.avro:avro:1.7.6-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.avro:avro:1.7.6-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: com.thoughtworks.paranamer:paranamer:2.3" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: com.thoughtworks.paranamer:paranamer:2.3" level="project" />
<orderEntry type="library" name="Maven: org.xerial.snappy:snappy-java:1.0.4.1" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.xerial.snappy:snappy-java:1.0.4.1" level="project" />
<orderEntry type="library" name="Maven: com.google.protobuf:protobuf-java:2.5.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.protobuf:protobuf-java:2.5.0" level="project" />
<orderEntry type="library" name="Maven: com.google.code.gson:gson:2.2.4" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.gson:gson:2.2.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-auth:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-auth:2.6.0-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.2.5" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.httpcomponents:httpclient:4.2.5" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.2.4" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.httpcomponents:httpcore:4.2.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-i18n:2.0.0-M15" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.directory.server:apacheds-i18n:2.0.0-M15" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.api:api-asn1-api:1.0.0-M20" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.directory.api:api-asn1-api:1.0.0-M20" level="project" />
<orderEntry type="library" name="Maven: org.apache.directory.api:api-util:1.0.0-M20" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.directory.api:api-util:1.0.0-M20" level="project" />
<orderEntry type="library" name="Maven: org.apache.curator:curator-framework:2.7.1" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.curator:curator-framework:2.7.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.curator:curator-client:2.7.1" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.curator:curator-client:2.7.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.curator:curator-recipes:2.7.1" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.curator:curator-recipes:2.7.1" level="project" />
<orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:3.0.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:jsr305:3.0.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.htrace:htrace-core4:4.0.1-incubating" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.htrace:htrace-core4:4.0.1-incubating" level="project" />
<orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.5-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.zookeeper:zookeeper:3.4.5-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.4.1" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.commons:commons-compress:1.4.1" level="project" />
<orderEntry type="library" name="Maven: org.tukaani:xz:1.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.tukaani:xz:1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-hdfs:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-hdfs:2.6.0-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: org.mortbay.jetty:jetty-util:6.1.26.cloudera.4" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.mortbay.jetty:jetty-util:6.1.26.cloudera.4" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty:3.6.2.Final" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: io.netty:netty:3.6.2.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-all:4.0.23.Final" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: io.netty:netty-all:4.0.23.Final" level="project" />
<orderEntry type="library" name="Maven: xerces:xercesImpl:2.9.1" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: xerces:xercesImpl:2.9.1" level="project" />
<orderEntry type="library" name="Maven: xml-apis:xml-apis:1.3.04" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: xml-apis:xml-apis:1.3.04" level="project" />
<orderEntry type="library" name="Maven: org.fusesource.leveldbjni:leveldbjni-all:1.8" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.fusesource.leveldbjni:leveldbjni-all:1.8" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-app:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-app:2.6.0-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-common:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-common:2.6.0-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-client:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-yarn-client:2.6.0-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-server-common:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-yarn-server-common:2.6.0-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-shuffle:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-shuffle:2.6.0-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-api:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-yarn-api:2.6.0-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-core:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-core:2.6.0-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-common:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-yarn-common:2.6.0-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: javax.xml.bind:jaxb-api:2.2.2" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: javax.xml.bind:jaxb-api:2.2.2" level="project" />
<orderEntry type="library" name="Maven: javax.xml.stream:stax-api:1.0-2" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: javax.xml.stream:stax-api:1.0-2" level="project" />
<orderEntry type="library" name="Maven: javax.activation:activation:1.1" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: javax.activation:activation:1.1" level="project" />
<orderEntry type="library" name="Maven: javax.servlet:servlet-api:2.5" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: javax.servlet:servlet-api:2.5" level="project" />
<orderEntry type="library" name="Maven: com.sun.jersey:jersey-core:1.9" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: com.sun.jersey:jersey-core:1.9" level="project" />
<orderEntry type="library" name="Maven: com.sun.jersey:jersey-client:1.9" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: com.sun.jersey:jersey-client:1.9" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-jaxrs:1.8.8" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.codehaus.jackson:jackson-jaxrs:1.8.8" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-xc:1.8.8" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.codehaus.jackson:jackson-xc:1.8.8" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.6.0-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-aws:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-aws:2.6.0-cdh5.7.0" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.2.3" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.2.3" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.2.3" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: com.fasterxml.jackson.core:jackson-core:2.2.3" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.2.3" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.2.3" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-s3:1.10.6" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: com.amazonaws:aws-java-sdk-s3:1.10.6" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-kms:1.10.6" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: com.amazonaws:aws-java-sdk-kms:1.10.6" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-core:1.10.6" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: com.amazonaws:aws-java-sdk-core:1.10.6" level="project" />
<orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-annotations:2.6.0-cdh5.7.0" level="project" /> <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.hadoop:hadoop-annotations:2.6.0-cdh5.7.0" level="project" />
</component> </component>
</module> </module>

91
pom.xml
View File

@@ -1,47 +1,66 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.fjy.hadoop</groupId> <groupId>com.fjy.hadoop</groupId>
<artifactId>hadoopstudy</artifactId> <artifactId>hadoopstudy</artifactId>
<version>0.4</version> <version>0.6</version>
<packaging>jar</packaging> <packaging>jar</packaging>
<name>hadoopstudy</name> <name>hadoopstudy</name>
<url>http://maven.apache.org</url> <url>http://maven.apache.org</url>
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<hadoop.version>2.6.0-cdh5.7.0</hadoop.version> <hadoop.version>2.6.0-cdh5.7.0</hadoop.version>
</properties> </properties>
<repositories> <repositories>
<repository> <repository>
<id>cloudera</id> <id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url> <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository> </repository>
</repositories> </repositories>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>
<artifactId>junit</artifactId> <artifactId>junit</artifactId>
<version>4.10</version> <version>4.10</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.kumkee</groupId> <groupId>com.kumkee</groupId>
<artifactId>UserAgentParser</artifactId> <artifactId>UserAgentParser</artifactId>
<version>0.0.1</version> <version>0.0.1</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId> <artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version> <version>${hadoop.version}</version>
</dependency> <scope>provided</scope>
</dependencies> </dependency>
</dependencies>
<!-- mvn assembly:assembly-->
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<archive>
<manifest>
<mainClass></mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
</plugins>
</build>
</project> </project>

162
src/main/java/com/fjy/hadoop/hadoop/project/LogApp.java Normal file
View File

@@ -0,0 +1,162 @@
package com.fjy.hadoop.hadoop.project;
import com.kumkee.userAgent.UserAgent;
import com.kumkee.userAgent.UserAgentParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
/**
* @author F嘉阳
* @date 2018-04-20 22:32
* 使用MapReduce完成日志分析和统计
*/
public class LogApp {
/**
* Map：读取输入文件
* Text:类似字符串
*/
public static class MyMapper extends Mapper<LongWritable, Text, Text, LongWritable> {
LongWritable one = new LongWritable(1);
private UserAgentParser userAgentParser;
@Override
protected void setup(Context context) throws IOException, InterruptedException {
userAgentParser = new UserAgentParser();
}
/**
* @param key 偏移量
* @param value 每行的字符串
* @param context 上下文
* @throws IOException
* @throws InterruptedException
*/
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
/*super.map(key, value, context);*/
//接收到每一行日志记录的数据
String source = value.toString();
String mobileuser = "";
UserAgent agent = userAgentParser.parse(source);
//测试
String browser = agent.getBrowser();
String engine = agent.getEngine();
String engineVersion = agent.getEngineVersion();
String os = agent.getOs();
String platform = agent.getPlatform();
boolean mobile = agent.isMobile();
if (mobile){
mobileuser = "手机用户";
}else {
mobileuser = "非手机用户";
}
//通过上下文把map的处理结果输出
context.write(new Text(browser), one);
context.write(new Text(engine), one);
context.write(new Text(engineVersion), one);
context.write(new Text(os), one);
context.write(new Text(platform), one);
context.write(new Text(mobileuser), one);
}
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
userAgentParser = null;
}
}
/**
* Reduce 归并操作
* LongWritable：文本出现的次数/求和后的次数
*/
public static class MyReducer extends Reducer<Text, LongWritable, Text, LongWritable> {
/**
* @param key
* @param values 相同偏移量的集合
* @param context
* @throws IOException
* @throws InterruptedException
*/
@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
//super.reduce(key, values, context);
long sum = 0;
for (LongWritable value : values) {
//求key出现的次数和总和
sum += value.get();
}
//统计结果的输出
context.write(key, new LongWritable(sum));
}
}
/**
* 定义Driver,封装MapReduce作业的所有信息
*/
public static void main(String[] args) throws Exception {
//创建Configuration
Configuration configuration = new Configuration();
//清理已存在的输出目录
Path outputPath = new Path(args[1]);
FileSystem fileSystem = FileSystem.get(configuration);
if (fileSystem.exists(outputPath)){
fileSystem.delete(outputPath);
System.out.println("The exist files had been deleted!");
}
//创建作业
Job job = Job.getInstance(configuration, "BrowserLogAccess");
//设置作业的主类
job.setJarByClass(LogApp.class);
//作业处理的输入路径
FileInputFormat.setInputPaths(job, new Path(args[0]));
//设置map相关参数
job.setMapperClass(LogApp.MyMapper.class);
//设置map输出的key的类型
job.setMapOutputKeyClass(Text.class);
//设置map输出的value的类型
job.setMapOutputValueClass(LongWritable.class);
//设置Reduce相关参数
job.setReducerClass(LogApp.MyReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
//设置作业处理输出结果的输出路径
FileOutputFormat.setOutputPath(job, new Path(args[1]));
//作业提交
job.waitForCompletion(true);
//作业完成后退出
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}