test/hadoopstudy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

MapReduce日志处理

Browse Source
This commit is contained in:
F嘉阳
2018-04-20 23:16:44 +08:00
parent 4623552076
commit 0301d1d924
4 changed files with 419 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

229
.idea/workspace.xml generated
View File

@@ -2,8 +2,10 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="8ccbe0a5-cdf8-4091-a84b-916013277899" name="Default" comment="">
<change beforePath="" afterPath="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/hadoop/project/LogApp.java" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change beforePath="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java" afterPath="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java" />
<change beforePath="$PROJECT_DIR$/hadoopstudy.iml" afterPath="$PROJECT_DIR$/hadoopstudy.iml" />
<change beforePath="$PROJECT_DIR$/pom.xml" afterPath="$PROJECT_DIR$/pom.xml" />
</list>
<ignored path="$PROJECT_DIR$/out/" />
<ignored path="$PROJECT_DIR$/target/" />
@@ -19,8 +21,8 @@
<file leaf-file-name="pom.xml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="410">
<caret line="40" column="43" lean-forward="false" selection-start-line="40" selection-start-column="43" selection-end-line="40" selection-end-column="43" />
<state relative-caret-position="138">
<caret line="6" column="16" lean-forward="false" selection-start-line="6" selection-start-column="16" selection-end-line="6" selection-end-column="16" />
<folding />
</state>
</provider>
@@ -29,11 +31,22 @@
<file leaf-file-name="UserAgentTest.java" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-629">
<caret line="23" column="23" lean-forward="true" selection-start-line="23" selection-start-column="23" selection-end-line="23" selection-end-column="23" />
<state relative-caret-position="364">
<caret line="23" column="20" lean-forward="false" selection-start-line="23" selection-start-column="20" selection-end-line="23" selection-end-column="20" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="LogApp.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/hadoop/project/LogApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="713">
<caret line="55" column="34" lean-forward="true" selection-start-line="55" selection-start-column="34" selection-end-line="55" selection-end-column="34" />
<folding>
<element signature="imports" expanded="true" />
<element signature="e#790#807#0" expanded="true" />
</folding>
</state>
</provider>
@@ -42,7 +55,7 @@
<file leaf-file-name="HDFSApp.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="207">
<state relative-caret-position="-2511">
<caret line="21" column="15" lean-forward="false" selection-start-line="21" selection-start-column="15" selection-end-line="21" selection-end-column="15" />
<folding>
<element signature="imports" expanded="true" />
@@ -54,8 +67,8 @@
<file leaf-file-name="WordCountApp.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="755">
<caret line="135" column="0" lean-forward="false" selection-start-line="135" selection-start-column="0" selection-end-line="135" selection-end-column="0" />
<state relative-caret-position="617">
<caret line="67" column="36" lean-forward="false" selection-start-line="67" selection-start-column="36" selection-end-line="67" selection-end-column="36" />
<folding>
<element signature="imports" expanded="true" />
</folding>
@@ -63,16 +76,6 @@
</provider>
</entry>
</file>
<file leaf-file-name="WordCountPartitionerApp.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountPartitionerApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="552">
<caret line="150" column="0" lean-forward="false" selection-start-line="150" selection-start-column="0" selection-end-line="150" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="Reducer.class" pinned="false" current-in-tab="false">
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Reducer.class">
<provider selected="true" editor-type-id="text-editor">
@@ -149,8 +152,9 @@
<option value="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java" />
<option value="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountPartitionerApp.java" />
<option value="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgent.java" />
<option value="$PROJECT_DIR$/pom.xml" />
<option value="$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java" />
<option value="$PROJECT_DIR$/pom.xml" />
<option value="$PROJECT_DIR$/src/main/java/com/fjy/hadoop/hadoop/project/LogApp.java" />
</list>
</option>
</component>
@@ -196,9 +200,8 @@
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scratches" />
<pane id="AndroidView" />
<pane id="Scope" />
<pane id="PackagesPane" />
<pane id="ProjectPane">
<subPane>
<expand>
@@ -230,6 +233,15 @@
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="hadoop" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="hadoopstudy" type="b2602c69:ProjectViewProjectNode" />
<item name="hadoopstudy" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="hadoop" type="462c0819:PsiDirectoryNode" />
<item name="mapreduce" type="462c0819:PsiDirectoryNode" />
</path>
<path>
@@ -261,7 +273,8 @@
<select />
</subPane>
</pane>
<pane id="Scratches" />
<pane id="Scope" />
<pane id="PackagesPane" />
</panes>
</component>
<component name="PropertiesComponent">
@@ -532,7 +545,8 @@
<workItem from="1523931905005" duration="6215000" />
<workItem from="1524056118416" duration="4576000" />
<workItem from="1524146174789" duration="602000" />
<workItem from="1524196421728" duration="4974000" />
<workItem from="1524196421728" duration="5000000" />
<workItem from="1524232464914" duration="3275000" />
</task>
<task id="LOCAL-00001" summary="建立初始工程">
<created>1521904777876</created>
@@ -597,13 +611,17 @@
<option name="project" value="LOCAL" />
<updated>1524214734524</updated>
</task>
<option name="localTasksCounter" value="10" />
<task id="LOCAL-00010" summary="单机版日志处理完成">
<created>1524215062883</created>
<option name="number" value="00010" />
<option name="presentableId" value="LOCAL-00010" />
<option name="project" value="LOCAL" />
<updated>1524215062883</updated>
</task>
<option name="localTasksCounter" value="11" />
<servers />
</component>
<component name="TestHistory">
<history-entry file="HDFSApp_copyToLocalFile - 2018.04.08 at 12h 00m 17s.xml">
<configuration name="HDFSApp.copyToLocalFile" configurationId="JUnit" />
</history-entry>
<history-entry file="HDFSApp_copyToLocalFile - 2018.04.08 at 12h 03m 40s.xml">
<configuration name="HDFSApp.copyToLocalFile" configurationId="JUnit" />
</history-entry>
@@ -631,9 +649,12 @@
<history-entry file="UserAgentTest_testReadFile - 2018.04.20 at 17h 03m 15s.xml">
<configuration name="UserAgentTest.testReadFile" configurationId="JUnit" />
</history-entry>
<history-entry file="UserAgentTest_testReadFile - 2018.04.20 at 23h 15m 30s.xml">
<configuration name="UserAgentTest.testReadFile" configurationId="JUnit" />
</history-entry>
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="37159000" />
<option name="totallyTimeSpent" value="40460000" />
</component>
<component name="ToolWindowManager">
<frame x="-8" y="-8" width="1936" height="1056" extended-state="6" />
@@ -648,19 +669,19 @@
<window_info id="Capture Analysis" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.31310943" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="true" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.31310943" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.1684435" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.17004265" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="UI Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Theme Preview" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.38159254" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
<window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
@@ -684,31 +705,14 @@
<MESSAGE value="使用Combiner加快处理效率" />
<MESSAGE value="使用Partitioner" />
<MESSAGE value="处理本地日志记录" />
<option name="LAST_COMMIT_MESSAGE" value="处理本地日志记录" />
<MESSAGE value="单机版日志处理完成" />
<option name="LAST_COMMIT_MESSAGE" value="单机版日志处理完成" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager />
<watches-manager />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="138">
<caret line="6" column="14" lean-forward="false" selection-start-line="6" selection-start-column="14" selection-end-line="6" selection-end-column="14" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="437">
<caret line="19" column="25" lean-forward="false" selection-start-line="19" selection-start-column="25" selection-end-line="19" selection-end-column="25" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Reducer.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="207">
@@ -1098,39 +1102,10 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/AppTest.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="115">
<caret line="7" column="28" lean-forward="false" selection-start-line="7" selection-start-column="28" selection-end-line="7" selection-end-column="28" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="755">
<caret line="135" column="0" lean-forward="false" selection-start-line="135" selection-start-column="0" selection-end-line="135" selection-end-column="0" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="207">
<caret line="21" column="15" lean-forward="false" selection-start-line="21" selection-start-column="15" selection-end-line="21" selection-end-column="15" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/Partitioner.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="184">
<caret line="11" column="7" lean-forward="true" selection-start-line="11" selection-start-column="7" selection-end-line="11" selection-end-column="7" />
<folding />
</state>
</provider>
</entry>
@@ -1138,23 +1113,6 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="298">
<caret line="496" column="16" lean-forward="false" selection-start-line="496" selection-start-column="16" selection-end-line="496" selection-end-column="16" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountPartitionerApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="552">
<caret line="150" column="0" lean-forward="false" selection-start-line="150" selection-start-column="0" selection-end-line="150" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="410">
<caret line="40" column="43" lean-forward="false" selection-start-line="40" selection-start-column="43" selection-end-line="40" selection-end-column="43" />
<folding />
</state>
</provider>
</entry>
@@ -1166,13 +1124,82 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java">
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountPartitionerApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-629">
<caret line="23" column="23" lean-forward="true" selection-start-line="23" selection-start-column="23" selection-end-line="23" selection-end-column="23" />
<state relative-caret-position="276">
<caret line="75" column="26" lean-forward="true" selection-start-line="75" selection-start-column="26" selection-end-line="75" selection-end-column="26" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/mapreduce/WordCountApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="617">
<caret line="67" column="36" lean-forward="false" selection-start-line="67" selection-start-column="36" selection-end-line="67" selection-end-column="36" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-common/2.6.0-cdh5.7.0/hadoop-common-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/io/Text.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="211">
<caret line="53" column="11" lean-forward="false" selection-start-line="53" selection-start-column="11" selection-end-line="53" selection-end-column="11" />
<folding />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-mapreduce-client-core/2.6.0-cdh5.7.0/hadoop-mapreduce-client-core-2.6.0-cdh5.7.0.jar!/org/apache/hadoop/mapreduce/TaskInputOutputContext.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="345">
<caret line="20" column="9" lean-forward="false" selection-start-line="20" selection-start-column="9" selection-end-line="20" selection-end-column="9" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="138">
<caret line="6" column="16" lean-forward="false" selection-start-line="6" selection-start-column="16" selection-end-line="6" selection-end-column="16" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/HDFSApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-2511">
<caret line="21" column="15" lean-forward="false" selection-start-line="21" selection-start-column="15" selection-end-line="21" selection-end-column="15" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/fjy/hadoop/hadoop/project/LogApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="713">
<caret line="55" column="34" lean-forward="true" selection-start-line="55" selection-start-column="34" selection-end-line="55" selection-end-column="34" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/AppTest.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="115">
<caret line="7" column="28" lean-forward="false" selection-start-line="7" selection-start-column="28" selection-end-line="7" selection-end-column="28" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/com/fjy/hadoop/UserAgentTest.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="364">
<caret line="23" column="20" lean-forward="false" selection-start-line="23" selection-start-column="20" selection-end-line="23" selection-end-column="20" />
<folding>
<element signature="imports" expanded="true" />
<element signature="e#790#807#0" expanded="true" />
</folding>
</state>
</provider>