Skip to content

S
SynRedis
Commit e6f6ccab authored by 杨书顺's avatar 杨书顺
Browse files
Options

程序源代码

parent 6744735d
Show whitespace changes
Inline Side-by-side
Showing with 1322 additions and 0 deletions
.DS_Store 0 → 100644
View file @ e6f6ccab
File added
.idea/compiler.xml 0 → 100644
View file @ e6f6ccab
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="SynRedis" />
</profile>
</annotationProcessing>
</component>
</project>
\ No newline at end of file
.idea/dictionaries/larkbird.xml 0 → 100644
View file @ e6f6ccab
<component name="ProjectDictionaryState">
<dictionary name="larkbird" />
</component>
\ No newline at end of file
.idea/encodings.xml 0 → 100644
View file @ e6f6ccab
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$" charset="UTF-8" />
</component>
</project>
\ No newline at end of file
.idea/misc.xml 0 → 100644
View file @ e6f6ccab
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="FrameworkDetectionExcludesConfiguration">
<file type="web" url="file://$PROJECT_DIR$" />
</component>
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
\ No newline at end of file
.idea/sbt.xml 0 → 100644
View file @ e6f6ccab
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ScalaSbtSettings">
<option name="customVMPath" />
</component>
</project>
\ No newline at end of file
.idea/workspace.xml 0 → 100644
View file @ e6f6ccab
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="f5081b32-11f5-465a-b8ac-ac230f73400d" name="Default" comment="" />
<ignored path="$PROJECT_DIR$/target/" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FileEditorManager">
<leaf>
<file leaf-file-name="SynHive2Redis.java" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/SynHive2Redis.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="292">
<caret line="22" column="41" lean-forward="true" selection-start-line="22" selection-start-column="41" selection-end-line="22" selection-end-column="41" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="application.properties" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/resources/application.properties">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="119">
<caret line="7" column="19" lean-forward="true" selection-start-line="7" selection-start-column="19" selection-end-line="7" selection-end-column="19" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="HiveUtil.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/HiveUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="109">
<caret line="9" column="21" selection-start-line="9" selection-start-column="13" selection-end-line="9" selection-end-column="21" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="pom.xml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="262">
<caret line="44" column="36" selection-start-line="44" selection-start-column="36" selection-end-line="44" selection-end-column="36" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="HadoopRedisUtil.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/HadoopRedisUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="399">
<caret line="339" column="52" selection-start-line="339" selection-start-column="49" selection-end-line="339" selection-end-column="52" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Class" />
</list>
</option>
</component>
<component name="Git.Settings">
<option name="ROOT_SYNC" value="DONT_SYNC" />
</component>
<component name="GradleLocalSettings">
<option name="projectSyncType">
<map>
<entry key="$PROJECT_DIR$/../../../opensource/kafka-eagle-master" value="PREVIEW" />
<entry key="$PROJECT_DIR$/../downloadcode/druid-service-master-1a7cb8de2481931f816c4a8aa463f83b70a98161" value="PREVIEW" />
<entry key="$PROJECT_DIR$/../downloadcode/duiba-jstorm-master-0020687997fcef0d1b91713cb9752f9b934029f5" value="PREVIEW" />
<entry key="$PROJECT_DIR$/../downloadcode/reports-develop-bacc3476d8a187e79ab32b0df80acc35d414376b" value="PREVIEW" />
<entry key="$PROJECT_DIR$/../druid-service" value="PREVIEW" />
<entry key="$PROJECT_DIR$/../duiba-jstorm" value="PREVIEW" />
<entry key="$PROJECT_DIR$/../product/duiba-jstorm" value="PREVIEW" />
<entry key="$PROJECT_DIR$/../reports" value="PREVIEW" />
</map>
</option>
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/PropertyUtil.java" />
<option value="$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/HiveUtil.java" />
<option value="$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/HadoopRedisUtil.java" />
<option value="$PROJECT_DIR$/src/main/java/com/duiba/synredis/SynHive2Redis.java" />
<option value="$PROJECT_DIR$/pom.xml" />
<option value="$PROJECT_DIR$/src/main/resources/application.properties" />
</list>
</option>
</component>
<component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsGulpfileManager">
<detection-done>true</detection-done>
<sorting>DEFINITION_ORDER</sorting>
</component>
<component name="MavenImportPreferences">
<option name="importingSettings">
<MavenImportingSettings>
<option name="importAutomatically" value="true" />
</MavenImportingSettings>
</option>
</component>
<component name="NodePackageJsonFileManager">
<packageJsonPaths />
</component>
<component name="ProjectFrameBounds" fullScreen="true">
<option name="y" value="23" />
<option name="width" value="1440" />
<option name="height" value="805" />
</component>
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="PackagesPane" />
<pane id="AndroidView" />
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="SynRedis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="SynRedis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="SynRedis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="SynRedis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="SynRedis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="synredis" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="SynRedis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="synredis" type="462c0819:PsiDirectoryNode" />
<item name="util" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="SynRedis" type="b2602c69:ProjectViewProjectNode" />
<item name="SynRedis" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="resources" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
<pane id="Scope" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="settings.editor.selected.configurable" value="settings.github" />
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/src/main/resources" />
<recent name="$PROJECT_DIR$/src/main/java/com/duiba/synredis/util" />
</key>
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="RunManager">
<configuration name="SynHive2Redis" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
<option name="MAIN_CLASS_NAME" value="com.duiba.synredis.SynHive2Redis" />
<module name="SynRedis" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.duiba.synredis.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
</configuration>
<configuration default="true" type="Application" factoryName="Application">
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
</configuration>
<configuration default="true" type="JUnit" factoryName="JUnit">
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="PACKAGE_NAME" />
<option name="MAIN_CLASS_NAME" />
<option name="METHOD_NAME" />
<option name="TEST_OBJECT" value="class" />
<option name="VM_PARAMETERS" value="-ea" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" value="%MODULE_WORKING_DIR%" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<patterns />
</configuration>
<configuration default="true" type="TestNG" factoryName="TestNG">
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="SUITE_NAME" />
<option name="PACKAGE_NAME" />
<option name="MAIN_CLASS_NAME" />
<option name="METHOD_NAME" />
<option name="GROUP_NAME" />
<option name="TEST_OBJECT" value="CLASS" />
<option name="VM_PARAMETERS" value="-ea" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" value="%MODULE_WORKING_DIR%" />
<option name="OUTPUT_DIRECTORY" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<option name="USE_DEFAULT_REPORTERS" value="false" />
<option name="PROPERTIES_FILE" />
<properties />
<listeners />
</configuration>
<recent_temporary>
<list>
<item itemvalue="Application.SynHive2Redis" />
</list>
</recent_temporary>
</component>
<component name="SvnConfiguration">
<configuration>$USER_HOME$/.subversion</configuration>
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="f5081b32-11f5-465a-b8ac-ac230f73400d" name="Default" comment="" />
<created>1531648879407</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1531648879407</updated>
<workItem from="1531648883734" duration="5942000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="5942000" />
</component>
<component name="ToolWindowManager">
<frame x="0" y="0" width="1440" height="900" extended-state="0" />
<editor active="true" />
<layout>
<window_info anchor="right" id="Palette" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Messages" sideWeight="0.49928468" weight="0.32969698" />
<window_info anchor="right" id="Palette&#9;" />
<window_info id="Image Layers" />
<window_info anchor="bottom" id="Java Enterprise" />
<window_info anchor="right" id="Capture Analysis" />
<window_info anchor="bottom" id="Event Log" sideWeight="0.5007153" side_tool="true" weight="0.32969698" />
<window_info anchor="right" id="Maven Projects" />
<window_info anchor="bottom" id="Database Changes" show_stripe_button="false" />
<window_info anchor="bottom" id="Run" order="2" weight="0.32969698" />
<window_info anchor="bottom" id="Version Control" show_stripe_button="false" />
<window_info anchor="bottom" id="Terminal" visible="true" weight="0.2969697" />
<window_info id="Capture Tool" />
<window_info id="Designer" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.24964234" />
<window_info id="Web" side_tool="true" />
<window_info anchor="right" id="Database" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info id="UI Designer" />
<window_info anchor="right" id="Theme Preview" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info id="Favorites" side_tool="true" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<option name="time" value="1" />
</breakpoint-manager>
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/.idea/workspace.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-4784">
<caret line="35" column="12" selection-start-line="35" selection-start-column="12" selection-end-line="35" selection-end-column="12" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/PropertyUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="153">
<caret line="14" column="55" selection-start-line="14" selection-start-column="44" selection-end-line="14" selection-end-column="55" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/HadoopRedisUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="399">
<caret line="339" column="52" selection-start-line="339" selection-start-column="49" selection-end-line="339" selection-end-column="52" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="262">
<caret line="44" column="36" selection-start-line="44" selection-start-column="36" selection-end-line="44" selection-end-column="36" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/util/HiveUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="109">
<caret line="9" column="21" selection-start-line="9" selection-start-column="13" selection-end-line="9" selection-end-column="21" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/application.properties">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="119">
<caret line="7" column="19" lean-forward="true" selection-start-line="7" selection-start-column="19" selection-end-line="7" selection-end-column="19" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/com/duiba/synredis/SynHive2Redis.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="292">
<caret line="22" column="41" lean-forward="true" selection-start-line="22" selection-start-column="41" selection-end-line="22" selection-end-column="41" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</component>
<component name="masterDetails">
<states>
<state key="ProjectJDKs.UI">
<settings>
<last-edited>1.8</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
<state key="ScopeChooserConfigurable.UI">
<settings>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
</states>
</component>
</project>
\ No newline at end of file
pom.xml 0 → 100644
View file @ e6f6ccab
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.duiba.synredis</groupId>
<artifactId>syn2redis</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>syn2redis Maven Webapp</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-jdbc</artifactId>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.21</version>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.9.0</version>
</dependency>
</dependencies>
<build>
<finalName>syn2redis</finalName>
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
<plugins>
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>3.0.0</version>
</plugin>
<!-- see http://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_war_packaging -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20.1</version>
</plugin>
<plugin>
<artifactId>maven-war-plugin</artifactId>
<version>3.2.0</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>
src/main/java/com/duiba/synredis/SynHive2Redis.java 0 → 100644
View file @ e6f6ccab
package com.duiba.synredis;
import com.duiba.synredis.util.HadoopRedisUtil;
import com.duiba.synredis.util.HiveUtil;
import java.util.HashMap;
import java.util.Map;
public class SynHive2Redis {
private final static String redisPre = "T51_";
private static HiveUtil hiveUtil = new HiveUtil();
private static HadoopRedisUtil redisUtil = HadoopRedisUtil.getInstance();
private Map<String, String> getAdvertTagData() {
String sql = "select advert_id,new_trate from advert.dws_advert_newtrate_df";
return hiveUtil.getMapData(sql);
}
public static void main(String[] args) {
SynHive2Redis synOpt = new SynHive2Redis();
Map<String, String> map = synOpt.getAdvertTagData();
Map<String, String> redisMap = new HashMap<>();
for (String key : map.keySet()) {
redisMap.put(redisPre + key, map.get(key));
}
if (redisMap.size() > 0) {
redisUtil.pipeLineMset(redisMap);
}
}
}
src/main/java/com/duiba/synredis/util/HadoopRedisUtil.java 0 → 100644
View file @ e6f6ccab
package com.duiba.synredis.util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class HadoopRedisUtil {
private static final Logger logger = LoggerFactory.getLogger(HadoopRedisUtil.class);
private static JedisPool jedisPool = null;
//volatile 禁止jvm进行指令重排
private volatile static HadoopRedisUtil instance = null;
private HadoopRedisUtil(int maxActive) {
//加载配置文件
PropertyUtil instance = PropertyUtil.getInstance();
if (maxActive == 0) {
maxActive = instance.getPropertyForInt("redis.max_active");
}
//创建redis连接池
JedisPoolConfig config = new JedisPoolConfig();
config.setMaxIdle(instance.getPropertyForInt("redis.max_idle"));
config.setMinIdle(instance.getPropertyForInt("redis.min_idle"));
config.setMaxWaitMillis(instance.getPropertyForLong("redis.max_wait"));
config.setMaxTotal(maxActive);
config.setTestOnBorrow(instance.getPropertyForBoolean("redis.test_on_borrow"));
config.setTestWhileIdle(true);
config.setTimeBetweenEvictionRunsMillis(instance.getPropertyForLong("redis.test_while_idle"));
jedisPool = new JedisPool(config, instance.getProperty("hadoop.redis.host"), instance.getPropertyForInt("redis.port"),
instance.getPropertyForInt("redis.timeout"), instance.getProperty("hadoop.redis.auth"));
}
public static HadoopRedisUtil getInstance() {
return getInstance(0);
}
public static HadoopRedisUtil getInstance(int maxActive) {
if (instance == null) {
init(maxActive);
}
return instance;
}
private synchronized static void init(int maxActive) {
if (instance == null) {
instance = new HadoopRedisUtil(maxActive);
}
}
/**
* 获取Jedis实例
*
* @return redis连接
*/
private Jedis getJedis() {
return jedisPool.getResource();
}
/**
* 释放jedis资源
*
* @param jedis redis连接
*/
private void returnResource(Jedis jedis) {
try {
if (jedis != null) {
jedis.close();
}
} catch (Exception e) {
logger.error("returnResource error.", e);
}
}
/**
* 返回hash中指定存储位置的值
*
* @param key 键值
* @param fieid 存储的名字
* @return 存储对应的值
*/
public String hget(String key, String fieid) {
Jedis jedis = getJedis();
try {
return jedis.hget(key, fieid);
} catch (Exception e) {
logger.error("redis hget error.", e);
} finally {
returnResource(jedis);
}
return null;
}
/**
* 添加对应关系，如果对应关系已存在，则覆盖
*
* @param key 键值
* @param field map中的key
* @param value map中的value
*/
public void hset(String key, String field, String value) {
Jedis jedis = getJedis();
try {
jedis.hset(key, field, value);
} catch (Exception e) {
logger.error("redis hmset error.", e);
} finally {
returnResource(jedis);
}
}
/**
* 添加对应关系，如果对应关系已存在，则覆盖
*
* @param key 键值
* @param map 存入redis中的map字段
*/
public void hmset(String key, Map<String, String> map) {
Jedis jedis = getJedis();
try {
jedis.hmset(key, map);
} catch (Exception e) {
logger.error("redis hmset error.", e);
} finally {
returnResource(jedis);
}
}
/**
* 以Map的形式返回hash中的存储和值
*
* @param key 键值
* @return map对象
*/
public Map<String, String> hgetAll(String key) {
Jedis jedis = getJedis();
try {
return jedis.hgetAll(key);
} catch (Exception e) {
logger.error("redis hgetAll error.", e);
} finally {
returnResource(jedis);
}
return null;
}
/**
* 计数器 指定key field 累计
*
* @param key 键值
* @return map对象
*/
public void hincrBy(String key, String field, Long value) {
Jedis jedis = getJedis();
try {
jedis.hincrBy(key, field, value);
} catch (Exception e) {
logger.error("redis hincrBy error.", e);
} finally {
returnResource(jedis);
}
}
/**
* 计数器 指定key field 累计
* 并设置生效时间
*
* @param key 键值
* @return map对象
*/
public void hincrBy(String key, String field, Long value, int seconds) {
Jedis jedis = getJedis();
try {
jedis.hincrBy(key, field, value);
jedis.expire(key, seconds);
} catch (Exception e) {
logger.error("redis hincrBy error.", e);
} finally {
returnResource(jedis);
}
}
/**
* 计数器 批量计数
*
* @param key 键值
* @return map对象
*/
public void hincrBy(String key, Map<String, Long> map) {
Jedis jedis = getJedis();
try {
Pipeline pipe = jedis.pipelined();
for (Map.Entry<String, Long> entry : map.entrySet()) {
pipe.hincrBy(key, entry.getKey(), entry.getValue());
}
pipe.sync();
} catch (Exception e) {
logger.error("redis hincrBy error.", e);
} finally {
returnResource(jedis);
}
}
/**
* 计数器 批量计数
*
* @param key 键值
* @return map对象
*/
public void hincrBy(String key, Map<String, Long> map, int seconds) {
Jedis jedis = getJedis();
try {
Pipeline pipe = jedis.pipelined();
for (Map.Entry<String, Long> entry : map.entrySet()) {
pipe.hincrBy(key, entry.getKey(), entry.getValue());
}
pipe.expire(key, seconds);
pipe.sync();
} catch (Exception e) {
logger.error("redis hincrBy error.", e);
} finally {
returnResource(jedis);
}
}
/**
* 计数器 批量计数
*
* @param redisKeyList redisKey集合
* @return map对象
*/
public void hincrBy(List<String> redisKeyList, Map<String, Long> map) {
Jedis jedis = getJedis();
try {
Pipeline pipe = jedis.pipelined();
for (String redisKey : redisKeyList) {
for (String column : map.keySet()) {
pipe.hincrBy(redisKey, column, map.get(column));
}
}
pipe.sync();
} catch (Exception e) {
logger.error("redis hincrBy error.", e);
} finally {
returnResource(jedis);
}
}
/**
* 判断key是否存在
*
* @param key 键值
* @return 是否存在 true表示存在
*/
public Boolean exists(String key) {
Jedis jedis = getJedis();
try {
return jedis.exists(key);
} catch (Exception e) {
logger.error("redis exists error.", e);
} finally {
returnResource(jedis);
}
return null;
}
/**
* 设置key的过期时间
*
* @param key 键值
* @param seconds 秒
* @return 设置成功返回 1 否则返回 0
*/
public Long expire(String key, int seconds) {
Jedis jedis = getJedis();
try {
return jedis.expire(key, seconds);
} catch (Exception e) {
logger.error("redis expire error.", e);
} finally {
returnResource(jedis);
}
return null;
}
/**
* 设置key的过期时间
*
* @param redisKeyList rediskey集合
* @param seconds 秒
*/
public void expire(List<String> redisKeyList, int seconds) {
Jedis jedis = getJedis();
try {
Pipeline pipe = jedis.pipelined();
for (String redisKey : redisKeyList) {
pipe.expire(redisKey, seconds);
}
pipe.sync();
} catch (Exception e) {
logger.error("redis expire error.", e);
} finally {
returnResource(jedis);
}
}
/**
* 通过pipeline 批量插入数据
*
* @param map key,value 键值对
* @param seconds 过期时间（单位：秒）
*/
public void pipeLineMset(Map<String, String> map, int seconds) {
Jedis jedis = getJedis();
try {
if (map != null && map.size() > 0) {
Pipeline pipeline = jedis.pipelined();
for (String key : map.keySet()) {
pipeline.setex(key, seconds, map.get(key));
}
pipeline.sync();
}
} catch (Exception e) {
logger.error("redis pipeLineMset error.", e);
} finally {
returnResource(jedis);
}
}
/**
* 通过pipeline 批量插入数据
* 永久写入
* @param map key,value 键值对
*/
public void pipeLineMset(Map<String, String> map) {
Jedis jedis = getJedis();
try {
if (map != null && map.size() > 0) {
Pipeline pipeline = jedis.pipelined();
for (String key : map.keySet()) {
pipeline.set(key, map.get(key));
}
pipeline.sync();
}
} catch (Exception e) {
logger.error("redis pipeLineMset error.", e);
} finally {
returnResource(jedis);
}
}
/**
* 批量删除key
*
* @param keyList 需要删除的key集合
*/
public void pipeLineDel(List<String> keyList) {
Jedis jedis = getJedis();
try {
if (keyList != null && keyList.size() > 0) {
Pipeline pipeline = jedis.pipelined();
for (String key : keyList) {
pipeline.del(key);
}
pipeline.sync();
}
} catch (Exception e) {
logger.error("redis pipeLineDel error.", e);
} finally {
returnResource(jedis);
}
}
/**
* 通过key获取redis中数据
*
* @param key 键值
* @return value
*/
public String get(String key) {
Jedis jedis = getJedis();
try {
return jedis.get(key);
} catch (Exception e) {
logger.error("redis get error.", e);
} finally {
returnResource(jedis);
}
return null;
}
/**
* 通过key写入redis中数据
*
* @param key 键值
* @param value 值
* @return value
*/
public String set(String key, String value, int expire) {
Jedis jedis = getJedis();
try {
return jedis.setex(key, expire, value);
} catch (Exception e) {
logger.error("redis set error.", e);
} finally {
returnResource(jedis);
}
return null;
}
public Long del(String key) {
Jedis jedis = getJedis();
try {
return jedis.del(key);
} catch (Exception e) {
logger.error("redis del error.", e);
} finally {
returnResource(jedis);
}
return null;
}
/**
* 批量查询数据
*
* @param keyList 指定查询的key集合，一次查询1000条，大量数据分批调用
* @return
*/
public Map<String, Map<String, String>> hmget(List<String> keyList) {
Map<String, Map<String, String>> resultMap = new HashMap<>();
Jedis jedis = getJedis();
try {
Pipeline pipe = jedis.pipelined();
Map<String, Response<Map<String, String>>> tmpMap = new HashMap<>();
for (String key : keyList) {
tmpMap.put(key, pipe.hgetAll(key));
}
pipe.sync();
for (Map.Entry<String, Response<Map<String, String>>> entry : tmpMap.entrySet()) {
// 解析map jie结果值
resultMap.put(entry.getKey(), entry.getValue().get());
}
} catch (Exception e) {
logger.error("redis hmget error.", e);
} finally {
returnResource(jedis);
}
return resultMap;
}
/**
* 通过key存入值
*
* @param key
* @param value
* @return
*/
public String set(String key, String value) {
Jedis jedis = getJedis();
try {
return jedis.set(key, value);
} catch (Exception e) {
logger.error("redis get error.", e);
} finally {
returnResource(jedis);
}
return null;
}
/**
* 批量查询数据
*
* @param keyList 指定查询的key集合，一次查询1000条，大量数据分批调用
* @return
*/
public Map<String, String> pipeLineMget(List<String> keyList) {
Map<String, String> resultMap = new HashMap<>();
Jedis jedis = getJedis();
try {
Pipeline pipe = jedis.pipelined();
Map<String, Response<String>> tmpMap = new HashMap<>();
for (String key : keyList) {
tmpMap.put(key, pipe.get(key));
}
pipe.sync();
for (Map.Entry<String, Response<String>> entry : tmpMap.entrySet()) {
// 解析map jie结果值
resultMap.put(entry.getKey(), entry.getValue().get());
}
} catch (Exception e) {
logger.error("redis pipeLineMget error.", e);
} finally {
returnResource(jedis);
}
return resultMap;
}
public void pipeLineHincrBy(String field, Map<String, Long> map, int seconds) {
Jedis jedis = getJedis();
try {
Pipeline pipe = jedis.pipelined();
for (Map.Entry<String, Long> entry : map.entrySet()) {
String key = entry.getKey();
pipe.hincrBy(key, field, entry.getValue());
pipe.expire(key, seconds);
}
pipe.sync();
} catch (Exception e) {
logger.error("redis hincrBy error.", e);
} finally {
returnResource(jedis);
}
}
}
src/main/java/com/duiba/synredis/util/HiveUtil.java 0 → 100644
View file @ e6f6ccab
package com.duiba.synredis.util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.*;
import java.util.HashMap;
import java.util.Map;
public class HiveUtil {
private static final Logger logger = LoggerFactory.getLogger(HiveUtil.class);
private String driverName;
private String url;
private String username;
private String pwd;
private Connection conn = null;
public HiveUtil() {
//加载配置文件
PropertyUtil instance = PropertyUtil.getInstance();
if (null != instance) {
driverName = instance.getProperty("hive.drivername");
url = instance.getProperty("hive.url");
username = instance.getProperty("hive.username");
pwd = instance.getProperty("hive.password");
}
}
public Connection getConnnection() {
try {
Class.forName(driverName);
conn = DriverManager.getConnection(url, username, pwd);
} catch (ClassNotFoundException e) {
e.printStackTrace();
System.exit(1);
} catch (SQLException e) {
logger.error("getConnection error" + e.getMessage());
e.printStackTrace();
}
return conn;
}
public static PreparedStatement prepare(Connection conn, String sql) {
PreparedStatement ps = null;
try {
ps = conn.prepareStatement(sql);
} catch (SQLException e) {
e.printStackTrace();
}
return ps;
}
public Map<String, String> getMapData(String sql) {
Map<String, String> map = new HashMap<>();
Connection connection = getConnnection();
PreparedStatement ps = prepare(connection, sql);
ResultSet rs =null;
try {
rs = ps.executeQuery();
int columns = rs.getMetaData().getColumnCount();
for (int i = 0; i < columns; i++) {
String colName = rs.getMetaData().getColumnName(i);
String colVal = rs.getString(i);
map.put(colName, colVal);
}
} catch (Exception e){
logger.error("getMapData error is "+e.getMessage());
} finally {
try {
rs.close();
ps.close();
connection.close();
}
catch (Exception e){
}
}
return map;
}
}
src/main/java/com/duiba/synredis/util/PropertyUtil.java 0 → 100644
View file @ e6f6ccab
package com.duiba.synredis.util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
public class PropertyUtil {
private static final Logger logger = LoggerFactory.getLogger(PropertyUtil.class);
//volatile 禁止jvm进行指令重排
private volatile static PropertyUtil instance = null;
private Properties props;
private static final String fileName = "application.properties";
private PropertyUtil(){
props = new Properties();
InputStream in = null;
try {
in = PropertyUtil.class.getClassLoader().getResourceAsStream(fileName);
props.load(in);
} catch (Exception e){
logger.error("PropertyUtil init error.", e);
} finally {
try {
if(in != null) {
in.close();
}
} catch (IOException e) {
logger.error("close file InputStream error.", e);
}
}
}
public static PropertyUtil getInstance(){
if(instance == null){
init();
}
return instance;
}
private synchronized static void init(){
if(instance == null){
instance = new PropertyUtil();
}
}
public String getProperty(String key){
if(props == null) {
init();
}
return props.getProperty(key);
}
public int getPropertyForInt(String key){
return Integer.parseInt(getProperty(key));
}
public long getPropertyForLong(String key){
return Long.parseLong(getProperty(key));
}
public boolean getPropertyForBoolean(String key){
return Boolean.parseBoolean(getProperty(key));
}
}
src/main/resources/application.properties 0 → 100644
View file @ e6f6ccab
#redis config
#common redis config
redis.port=6379
redis.max_active=10
redis.max_idle=5
redis.min_idle=1
redis.max_wait=10000
redis.timeout=10000
redis.test_while_idle=90000
redis.test_on_borrow=false
hadoop.redis.host=r-bp11df1c1400afa4.redis.rds.aliyuncs.com
hadoop.redis.auth=UjTD4apxUgu4xNVTnRAtqQt
#hive config
hive.drivername=org.apache.hive.jdbc.HiveDriver
hive.url=jdbc:hive2://172.16.1.21:10000/default
hive.username=hive
hive.password=hive
\ No newline at end of file
target/classes/application.properties 0 → 100644
View file @ e6f6ccab
#redis config
#common redis config
#redis\u8BBF\u95EE\u7AEF\u53E3
redis.port=6379
#\u6700\u5927\u53EF\u7528\u8FDE\u63A5\u6570\uFF0C\u5982\u679C\u8D4B\u503C\u4E3A-1\uFF0C\u5219\u8868\u793A\u4E0D\u9650\u5236\uFF1B\u5982\u679Cpool\u5DF2\u7ECF\u5206\u914D\u4E86maxActive\u4E2Ajedis\u5B9E\u4F8B\uFF0C\u5219\u6B64\u65F6pool\u7684\u72B6\u6001\u4E3Aexhausted(\u8017\u5C3D)\u3002
redis.max_active=10
#\u63A7\u5236\u4E00\u4E2Apool\u6700\u591A\u6709\u591A\u5C11\u4E2A\u72B6\u6001\u4E3Aidle(\u7A7A\u95F2\u7684)\u7684redis\u8FDE\u63A5
redis.max_idle=5
#\u63A7\u5236\u4E00\u4E2Apool\u6700\u5C11\u6709\u591A\u5C11\u4E2A\u72B6\u6001\u4E3Aidle(\u7A7A\u95F2\u7684)\u7684redis\u8FDE\u63A5
redis.min_idle=1
#\u7B49\u5F85\u53EF\u7528\u8FDE\u63A5\u7684\u6700\u5927\u65F6\u95F4\uFF0C\u5355\u4F4D\u6BEB\u79D2\uFF0C\u9ED8\u8BA4\u503C\u4E3A-1\uFF0C\u8868\u793A\u6C38\u4E0D\u8D85\u65F6\u3002\u5982\u679C\u8D85\u8FC7\u7B49\u5F85\u65F6\u95F4\uFF0C\u5219\u76F4\u63A5\u629B\u51FAJedisConnectionException\uFF1B
redis.max_wait=10000
#\u8FDE\u63A5\u7684\u8D85\u65F6\u65F6\u95F4
redis.timeout=10000
#\u6BCF\u969490S testWhileIdle\u4E00\u6B21
redis.test_while_idle=90000
#\u5728borrow\u4E00\u4E2Ajedis\u5B9E\u4F8B\u65F6\uFF0C\u662F\u5426\u63D0\u524D\u8FDB\u884Cvalidate\u64CD\u4F5C\uFF1B\u5982\u679C\u4E3Atrue\uFF0C\u5219\u5F97\u5230\u7684jedis\u5B9E\u4F8B\u5747\u662F\u53EF\u7528\u7684,\u4F46\u4F1A\u5F71\u54CD\u6027\u80FD
redis.test_on_borrow=false
#hadoop redis cluster (\u963F\u91CC\u4E9116G\u96C6\u7FA4)
hadoop.redis.host=r-bp11df1c1400afa4.redis.rds.aliyuncs.com
#\u8BBF\u95EE\u5BC6\u7801
hadoop.redis.auth=UjTD4apxUgu4xNVTnRAtqQt
#hive config
hive.drivername=org.apache.hive.jdbc.HiveDriver
hive.url=jdbc:hive2://172.16.1.21:10000/default
hive.username=hive
hive.password=hive
\ No newline at end of file
target/maven-archiver/pom.properties 0 → 100644
View file @ e6f6ccab
#Generated by Maven
#Sun Jul 15 18:13:42 CST 2018
version=1.0-SNAPSHOT
groupId=com.duiba.synredis
artifactId=syn2redis
target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst 0 → 100644
View file @ e6f6ccab
com/duiba/synredis/util/PropertyUtil.class
com/duiba/synredis/util/HadoopRedisUtil.class
com/duiba/synredis/util/HiveUtil.class
com/duiba/synredis/SynHive2Redis.class
target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst 0 → 100644
View file @ e6f6ccab
/Users/larkbird/Library/Mobile Documents/com~apple~CloudDocs/work/yss/project/SynRedis/src/main/java/com/duiba/synredis/SynHive2Redis.java
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment