Skip to content

S
SynRedis
Commit e1df994e authored by 杨书顺's avatar 杨书顺
Browse files
Options

测试同步hive数据到es表

parent e6f6ccab
Hide whitespace changes
Inline Side-by-side
Showing with 283 additions and 122 deletions
.idea/compiler.xml
View file @ e1df994e
......@@ -6,7 +6,7 @@
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="SynRedis" />
<module name="syn2redis" />
</profile>
</annotationProcessing>
</component>
......
.idea/misc.xml
View file @ e1df994e
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="FrameworkDetectionExcludesConfiguration">
<file type="web" url="file://$PROJECT_DIR$" />
</component>
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
......
.idea/workspace.xml
View file @ e1df994e
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="f5081b32-11f5-465a-b8ac-ac230f73400d" name="Default" comment="" />
<list default="true" id="e25f4488-2e3b-450b-8f4d-869369b95e9d" name="Default" comment="">
<change afterPath="$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/ElasticSearchUtil.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/compiler.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/compiler.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/misc.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/misc.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pom.xml" beforeDir="false" afterPath="$PROJECT_DIR$/pom.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/main/resources/application.properties" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/resources/application.properties" afterDir="false" />
</list>
<ignored path="$PROJECT_DIR$/target/" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
......@@ -11,33 +18,44 @@
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FileEditorManager">
<leaf>
<file leaf-file-name="SynHive2Redis.java" pinned="false" current-in-tab="true">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="SynHive2Redis.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/SynHive2Redis.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="292">
<caret line="22" column="41" lean-forward="true" selection-start-line="22" selection-start-column="41" selection-end-line="22" selection-end-column="41" />
<state relative-caret-position="187">
<caret line="15" column="40" selection-start-line="15" selection-start-column="40" selection-end-line="15" selection-end-column="40" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="synhive2es.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/test/synhive2es.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="408">
<caret line="24" lean-forward="true" selection-start-line="24" selection-end-line="24" />
<folding>
<element signature="imports" expanded="true" />
<element signature="e#1115#1116#0" expanded="true" />
<element signature="e#1142#1143#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="application.properties" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/resources/application.properties">
<file leaf-file-name="HiveUtil.java" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/HiveUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="119">
<caret line="7" column="19" lean-forward="true" selection-start-line="7" selection-start-column="19" selection-end-line="7" selection-end-column="19" />
<state relative-caret-position="486">
<caret line="73" column="27" lean-forward="true" selection-start-line="73" selection-start-column="27" selection-end-line="73" selection-end-column="27" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="HiveUtil.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/HiveUtil.java">
<file leaf-file-name="ElasticSearchUtil.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/ElasticSearchUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="109">
<caret line="9" column="21" selection-start-line="9" selection-start-column="13" selection-end-line="9" selection-end-column="21" />
<state relative-caret-position="204">
<caret line="12" column="21" lean-forward="true" selection-start-line="12" selection-start-column="21" selection-end-line="12" selection-end-column="21" />
<folding>
<element signature="imports" expanded="true" />
</folding>
......@@ -45,23 +63,20 @@
</provider>
</entry>
</file>
<file leaf-file-name="pom.xml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/pom.xml">
<file leaf-file-name="application.properties" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/resources/application.properties">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="262">
<caret line="44" column="36" selection-start-line="44" selection-start-column="36" selection-end-line="44" selection-end-column="36" />
<state relative-caret-position="323">
<caret line="19" lean-forward="true" selection-start-line="19" selection-end-line="19" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="HadoopRedisUtil.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/HadoopRedisUtil.java">
<file leaf-file-name="pom.xml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="399">
<caret line="339" column="52" selection-start-line="339" selection-start-column="49" selection-end-line="339" selection-end-column="52" />
<folding>
<element signature="imports" expanded="true" />
</folding>
<state relative-caret-position="548">
<caret line="75" column="21" selection-start-line="75" selection-start-column="21" selection-end-line="75" selection-end-column="21" />
</state>
</provider>
</entry>
......@@ -77,6 +92,7 @@
</component>
<component name="Git.Settings">
<option name="ROOT_SYNC" value="DONT_SYNC" />
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="GradleLocalSettings">
<option name="projectSyncType">
......@@ -95,12 +111,10 @@
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/PropertyUtil.java" />
<option value="$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/HiveUtil.java" />
<option value="$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/HadoopRedisUtil.java" />
<option value="$PROJECT_DIR$/src/main/java/com/duiba/synredis/SynHive2Redis.java" />
<option value="$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/ElasticSearchUtil.java" />
<option value="$PROJECT_DIR$/pom.xml" />
<option value="$PROJECT_DIR$/src/main/resources/application.properties" />
<option value="$PROJECT_DIR$/src/main/java/com/duiba/synredis/test/synhive2es.java" />
</list>
</option>
</component>
......@@ -120,45 +134,47 @@
<component name="NodePackageJsonFileManager">
<packageJsonPaths />
</component>
<component name="ProjectFrameBounds" fullScreen="true">
<component name="ProjectFrameBounds" extendedState="6" fullScreen="true">
<option name="y" value="23" />
<option name="width" value="1440" />
<option name="height" value="805" />
</component>
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="PackagesPane" />
<pane id="Scope" />
<pane id="AndroidView" />
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="SynRedis" type="b2602c69:ProjectViewProjectNode" />
<item name="syn2redis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="SynRedis" type="b2602c69:ProjectViewProjectNode" />
<item name="syn2redis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="SynRedis" type="b2602c69:ProjectViewProjectNode" />
<item name="syn2redis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="SynRedis" type="b2602c69:ProjectViewProjectNode" />
<item name="syn2redis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="SynRedis" type="b2602c69:ProjectViewProjectNode" />
<item name="syn2redis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
......@@ -166,7 +182,16 @@
<item name="synredis" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="SynRedis" type="b2602c69:ProjectViewProjectNode" />
<item name="syn2redis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="synredis" type="462c0819:PsiDirectoryNode" />
<item name="test" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="syn2redis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
......@@ -175,7 +200,7 @@
<item name="util" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="SynRedis" type="b2602c69:ProjectViewProjectNode" />
<item name="syn2redis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
......@@ -185,7 +210,6 @@
<select />
</subPane>
</pane>
<pane id="Scope" />
</panes>
</component>
<component name="PropertiesComponent">
......@@ -193,12 +217,11 @@
<property name="aspect.path.notification.shown" value="true" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="settings.editor.selected.configurable" value="settings.github" />
<property name="settings.editor.selected.configurable" value="vcs.Subversion.SSH" />
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/src/main/resources" />
<recent name="$PROJECT_DIR$/src/main/java/com/duiba/synredis/util" />
<key name="CopyClassDialog.RECENTS_KEY">
<recent name="com.duiba.synredis.util" />
</key>
</component>
<component name="RunDashboard">
......@@ -214,17 +237,6 @@
</option>
</component>
<component name="RunManager">
<configuration name="SynHive2Redis" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
<option name="MAIN_CLASS_NAME" value="com.duiba.synredis.SynHive2Redis" />
<module name="SynRedis" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.duiba.synredis.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
</configuration>
<configuration default="true" type="Application" factoryName="Application">
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
</configuration>
......@@ -266,63 +278,57 @@
<properties />
<listeners />
</configuration>
<recent_temporary>
<list>
<item itemvalue="Application.SynHive2Redis" />
</list>
</recent_temporary>
</component>
<component name="SvnConfiguration">
<configuration>$USER_HOME$/.subversion</configuration>
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="f5081b32-11f5-465a-b8ac-ac230f73400d" name="Default" comment="" />
<created>1531648879407</created>
<changelist id="e25f4488-2e3b-450b-8f4d-869369b95e9d" name="Default" comment="" />
<created>1531752556247</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1531648879407</updated>
<workItem from="1531648883734" duration="5942000" />
<updated>1531752556247</updated>
<workItem from="1531752561136" duration="638000" />
<workItem from="1532267467068" duration="2971000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="5942000" />
<option name="totallyTimeSpent" value="3609000" />
</component>
<component name="ToolWindowManager">
<frame x="0" y="0" width="1440" height="900" extended-state="0" />
<frame x="0" y="0" width="1440" height="900" extended-state="6" />
<editor active="true" />
<layout>
<window_info anchor="right" id="Palette" />
<window_info anchor="right" id="Palette" order="3" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Messages" sideWeight="0.49928468" weight="0.32969698" />
<window_info anchor="right" id="Palette&#9;" />
<window_info id="Image Layers" />
<window_info anchor="bottom" id="Java Enterprise" />
<window_info anchor="right" id="Capture Analysis" />
<window_info anchor="bottom" id="Event Log" sideWeight="0.5007153" side_tool="true" weight="0.32969698" />
<window_info anchor="right" id="Maven Projects" />
<window_info anchor="bottom" id="Database Changes" show_stripe_button="false" />
<window_info anchor="bottom" id="Run" order="2" weight="0.32969698" />
<window_info anchor="bottom" id="Version Control" show_stripe_button="false" />
<window_info anchor="bottom" id="Terminal" visible="true" weight="0.2969697" />
<window_info id="Capture Tool" />
<window_info id="Designer" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.24964234" />
<window_info id="Web" side_tool="true" />
<window_info anchor="right" id="Database" />
<window_info anchor="right" id="Palette&#9;" order="3" />
<window_info id="Image Layers" order="2" />
<window_info anchor="bottom" id="Java Enterprise" order="7" />
<window_info anchor="right" id="Capture Analysis" order="3" />
<window_info anchor="bottom" id="Event Log" order="7" side_tool="true" />
<window_info anchor="right" id="Maven Projects" order="3" />
<window_info anchor="bottom" id="Database Changes" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="Run" order="2" />
<window_info anchor="bottom" id="Version Control" order="7" weight="0.32909605" />
<window_info anchor="bottom" id="Terminal" order="7" />
<window_info id="Capture Tool" order="2" />
<window_info id="Designer" order="2" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.2532189" />
<window_info anchor="right" id="Database" order="3" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info id="UI Designer" />
<window_info anchor="right" id="Theme Preview" />
<window_info id="UI Designer" order="2" />
<window_info anchor="right" id="Theme Preview" order="3" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info id="Favorites" side_tool="true" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Find" order="1" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
......@@ -331,67 +337,61 @@
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<option name="time" value="1" />
</breakpoint-manager>
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/.idea/workspace.xml">
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/SynHive2Redis.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-4784">
<caret line="35" column="12" selection-start-line="35" selection-start-column="12" selection-end-line="35" selection-end-column="12" />
<state relative-caret-position="238">
<caret line="18" column="44" lean-forward="true" selection-start-line="18" selection-start-column="44" selection-end-line="18" selection-end-column="44" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/PropertyUtil.java">
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/SynHive2Redis.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="153">
<caret line="14" column="55" selection-start-line="14" selection-start-column="44" selection-end-line="14" selection-end-column="55" />
<state relative-caret-position="187">
<caret line="15" column="40" selection-start-line="15" selection-start-column="40" selection-end-line="15" selection-end-column="40" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/HadoopRedisUtil.java">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="399">
<caret line="339" column="52" selection-start-line="339" selection-start-column="49" selection-end-line="339" selection-end-column="52" />
<folding>
<element signature="imports" expanded="true" />
</folding>
<state relative-caret-position="548">
<caret line="75" column="21" selection-start-line="75" selection-start-column="21" selection-end-line="75" selection-end-column="21" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<entry file="file://$PROJECT_DIR$/src/main/resources/application.properties">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="262">
<caret line="44" column="36" selection-start-line="44" selection-start-column="36" selection-end-line="44" selection-end-column="36" />
<state relative-caret-position="323">
<caret line="19" lean-forward="true" selection-start-line="19" selection-end-line="19" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/HiveUtil.java">
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/test/synhive2es.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="109">
<caret line="9" column="21" selection-start-line="9" selection-start-column="13" selection-end-line="9" selection-end-column="21" />
<state relative-caret-position="408">
<caret line="24" lean-forward="true" selection-start-line="24" selection-end-line="24" />
<folding>
<element signature="imports" expanded="true" />
<element signature="e#1115#1116#0" expanded="true" />
<element signature="e#1142#1143#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/application.properties">
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/ElasticSearchUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="119">
<caret line="7" column="19" lean-forward="true" selection-start-line="7" selection-start-column="19" selection-end-line="7" selection-end-column="19" />
<state relative-caret-position="204">
<caret line="12" column="21" lean-forward="true" selection-start-line="12" selection-start-column="21" selection-end-line="12" selection-end-column="21" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/SynHive2Redis.java">
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/HiveUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="292">
<caret line="22" column="41" lean-forward="true" selection-start-line="22" selection-start-column="41" selection-end-line="22" selection-end-column="41" />
<folding>
<element signature="imports" expanded="true" />
</folding>
<state relative-caret-position="486">
<caret line="73" column="27" lean-forward="true" selection-start-line="73" selection-start-column="27" selection-end-line="73" selection-end-column="27" />
</state>
</provider>
</entry>
......
pom.xml
View file @ e1df994e
......@@ -51,6 +51,29 @@
<version>2.9.0</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>2.4.6</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>5.5.4</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.9.7</version>
</dependency>
<dependency>
<groupId>my.elasticsearch</groupId>
<artifactId>es-shaded</artifactId>
<version>1.0.1</version>
</dependency>
</dependencies>
<build>
......
src/main/java/com/duiba/synredis/test/synhive2es.java 0 → 100644
View file @ e1df994e
package com.duiba.synredis.test;
import org.apache.commons.lang3.StringUtils;
import java.util.Map;
/**
* 同步hive表tmp.es_tuia_advert_app_adslot_coupon_di 数据到es表es_tuia_advert_app_adslot_coupon_di
*/
public class synhive2es {
private String hivetable = "tmp.es_tuia_advert_app_adslot_coupon_di";
private String sourceFiled = "slot_id,start_time,finish_time,advert_id,app_id,launch_pv,exposure_pv,click_pv,consumer_total,pay_launch_pv,pay_exposure_pv,pay_click_pv";
private String sourceFiledType = "bigint,string,string,bigint,bigint,bigint,bigint,bigint,bigint,bigint,bigint,bigint";
private Map<String, String> getHiveData(String tableName, String filedList,String queryDate) {
if (StringUtils.isNotBlank(filedList)) {
String[] filedStrs = filedList.split(",");
StringBuffer colBuffer = new StringBuffer();
for (String col : filedStrs) {
colBuffer.append(col).append(",");
}
String selectSql = colBuffer.substring(0, colBuffer.length() - 1);
String sql = String.format("select %s from %s where dt='%s'",selectSql,tableName,queryDate);
}
return null;
}
private Map<String, Map<String, String>> getEsData() {
return null;
}
private void writeEs(String index, String type, Map<String, Map<String, Object>> map) {
}
}
src/main/java/com/duiba/synredis/util/ElasticSearchUtil.java 0 → 100644
View file @ e1df994e
package com.duiba.synredis.util;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.InetAddress;
import java.util.Map;
public class ElasticSearchUtil {
private static final Logger logger = LoggerFactory.getLogger(ElasticSearchUtil.class);
private volatile static ElasticSearchUtil instance = null;
private TransportClient client;
public static ElasticSearchUtil getInstance() {
if (instance == null) {
init();
}
return instance;
}
private synchronized static void init() {
if (instance == null) {
instance = new ElasticSearchUtil();
}
}
private ElasticSearchUtil() {
initClient();
}
private synchronized TransportClient initClient() {
if (client != null) {
return client;
}
try {
//加载配置文件
PropertyUtil instance = PropertyUtil.getInstance();
Settings settings = Settings.settingsBuilder().put("cluster.name", instance.getProperty("es.cluster.name"))
.put("client.transport.sniff", true).build();
String hostsports = instance.getProperty("es.hostsports");
if (StringUtils.isNotBlank(hostsports)) {
client = TransportClient.builder().settings(settings).build();
String[] hostsPortStrs = hostsports.split(",");
for (String hostport : hostsPortStrs) {
String[] strs = hostport.split(":");
client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(strs[0]), Integer.valueOf(strs[1])));
}
}
return client;
} catch (Exception e) {
logger.error("initClient error", e);
}
return null;
}
public void batchInsertDoc(String index, String type, Map<String, Map<String, Object>> map) {
try{
if (StringUtils.isNotBlank(index) && StringUtils.isNotBlank(type) && null != map) {
BulkRequestBuilder bulkRequest = client.prepareBulk();
for (String key : map.keySet()) {
bulkRequest.add(client.prepareIndex(index, type,key).setSource(map.get(key)));
}
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if (bulkResponse.hasFailures()) {
logger.error(bulkResponse.buildFailureMessage());
}
}
}
catch (Exception e){
logger.error("batchInsertDoc error.", e);
}
}
private void closeClient() {
try {
if (client != null) {
client.close();
}
} catch (Exception e) {
logger.error("closeClient error.", e);
}
}
public static void main(String[] args) throws Exception {
}
}
src/main/resources/application.properties
View file @ e1df994e
......@@ -16,4 +16,12 @@ hadoop.redis.auth=UjTD4apxUgu4xNVTnRAtqQt
hive.drivername=org.apache.hive.jdbc.HiveDriver
hive.url=jdbc:hive2://172.16.1.21:10000/default
hive.username=hive
hive.password=hive
\ No newline at end of file
hive.password=hive
#es cluster config
#es.cluster.name=bigdata-es
#es.hostsports=172.16.1.121:9300,172.16.1.122:9300,172.16.1.123:9300
#test-es cluster config
es.cluster.name=test-es
es.hostsports=172.16.1.141:9300,172.16.1.142:9300,172.16.1.143:9300
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment