程序源代码

.idea/compiler.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CompilerConfiguration">
+    <annotationProcessing>
+      <profile name="Maven default annotation processors profile" enabled="true">
+        <sourceOutputDir name="target/generated-sources/annotations" />
+        <sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
+        <outputRelativeToContentRoot value="true" />
+        <module name="SynRedis" />
+      </profile>
+    </annotationProcessing>
+  </component>
+</project>
\ No newline at end of file

.idea/dictionaries/larkbird.xml

+<component name="ProjectDictionaryState">
+  <dictionary name="larkbird" />
+</component>
\ No newline at end of file

.idea/encodings.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Encoding">
+    <file url="file://$PROJECT_DIR$" charset="UTF-8" />
+  </component>
+</project>
\ No newline at end of file

.idea/misc.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ExternalStorageConfigurationManager" enabled="true" />
+  <component name="FrameworkDetectionExcludesConfiguration">
+    <file type="web" url="file://$PROJECT_DIR$" />
+  </component>
+  <component name="MavenProjectsManager">
+    <option name="originalFiles">
+      <list>
+        <option value="$PROJECT_DIR$/pom.xml" />
+      </list>
+    </option>
+  </component>
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>
\ No newline at end of file

.idea/sbt.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ScalaSbtSettings">
+    <option name="customVMPath" />
+  </component>
+</project>
\ No newline at end of file

pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.duiba.synredis</groupId>
+    <artifactId>syn2redis</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <name>syn2redis Maven Webapp</name>
+    <!-- FIXME change it to the project's website -->
+    <url>http://www.example.com</url>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.source>1.7</maven.compiler.source>
+        <maven.compiler.target>1.7</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.11</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>2.6.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hive</groupId>
+            <artifactId>hive-jdbc</artifactId>
+            <version>2.2.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.21</version>
+        </dependency>
+
+        <dependency>
+            <groupId>redis.clients</groupId>
+            <artifactId>jedis</artifactId>
+            <version>2.9.0</version>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <finalName>syn2redis</finalName>
+        <pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
+            <plugins>
+                <plugin>
+                    <artifactId>maven-clean-plugin</artifactId>
+                    <version>3.0.0</version>
+                </plugin>
+                <!-- see http://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_war_packaging -->
+                <plugin>
+                    <artifactId>maven-resources-plugin</artifactId>
+                    <version>3.0.2</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-compiler-plugin</artifactId>
+                    <version>3.7.0</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-surefire-plugin</artifactId>
+                    <version>2.20.1</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-war-plugin</artifactId>
+                    <version>3.2.0</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-install-plugin</artifactId>
+                    <version>2.5.2</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-deploy-plugin</artifactId>
+                    <version>2.8.2</version>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+    </build>
+</project>

src/main/java/com/duiba/synredis/SynHive2Redis.java

+package com.duiba.synredis;
+
+import com.duiba.synredis.util.HadoopRedisUtil;
+import com.duiba.synredis.util.HiveUtil;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class SynHive2Redis {
+    private final static String redisPre = "T51_";
+    private static HiveUtil hiveUtil = new HiveUtil();
+    private static HadoopRedisUtil redisUtil = HadoopRedisUtil.getInstance();
+
+    private Map<String, String> getAdvertTagData() {
+        String sql = "select advert_id,new_trate from advert.dws_advert_newtrate_df";
+        return hiveUtil.getMapData(sql);
+    }
+
+    public static void main(String[] args) {
+        SynHive2Redis synOpt = new SynHive2Redis();
+        Map<String, String> map = synOpt.getAdvertTagData();
+        Map<String, String> redisMap = new HashMap<>();
+        for (String key : map.keySet()) {
+            redisMap.put(redisPre + key, map.get(key));
+        }
+        if (redisMap.size() > 0) {
+            redisUtil.pipeLineMset(redisMap);
+        }
+    }
+}

src/main/java/com/duiba/synredis/util/HiveUtil.java

+package com.duiba.synredis.util;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.*;
+import java.util.HashMap;
+import java.util.Map;
+
+public class HiveUtil {
+    private static final Logger logger = LoggerFactory.getLogger(HiveUtil.class);
+    private String driverName;
+    private String url;
+    private String username;
+    private String pwd;
+    private Connection conn = null;
+
+    public HiveUtil() {
+        //加载配置文件
+        PropertyUtil instance = PropertyUtil.getInstance();
+        if (null != instance) {
+            driverName = instance.getProperty("hive.drivername");
+            url = instance.getProperty("hive.url");
+            username = instance.getProperty("hive.username");
+            pwd = instance.getProperty("hive.password");
+        }
+    }
+
+    public Connection getConnnection() {
+        try {
+            Class.forName(driverName);
+            conn = DriverManager.getConnection(url, username, pwd);
+        } catch (ClassNotFoundException e) {
+            e.printStackTrace();
+            System.exit(1);
+        } catch (SQLException e) {
+            logger.error("getConnection error" + e.getMessage());
+            e.printStackTrace();
+        }
+        return conn;
+    }
+
+    public static PreparedStatement prepare(Connection conn, String sql) {
+        PreparedStatement ps = null;
+        try {
+            ps = conn.prepareStatement(sql);
+        } catch (SQLException e) {
+
+            e.printStackTrace();
+        }
+        return ps;
+    }
+
+
+    public Map<String, String> getMapData(String sql) {
+        Map<String, String> map = new HashMap<>();
+        Connection connection = getConnnection();
+        PreparedStatement ps = prepare(connection, sql);
+        ResultSet rs =null;
+        try {
+             rs = ps.executeQuery();
+            int columns = rs.getMetaData().getColumnCount();
+            for (int i = 0; i < columns; i++) {
+                String colName = rs.getMetaData().getColumnName(i);
+                String colVal = rs.getString(i);
+                map.put(colName, colVal);
+            }
+
+        } catch (Exception e){
+           logger.error("getMapData error is "+e.getMessage());
+        } finally {
+            try {
+                rs.close();
+                ps.close();
+                connection.close();
+            }
+            catch (Exception e){
+            }
+        }
+        return map;
+    }
+
+
+}

src/main/java/com/duiba/synredis/util/PropertyUtil.java

+package com.duiba.synredis.util;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+public class PropertyUtil {
+    private static final Logger logger = LoggerFactory.getLogger(PropertyUtil.class);
+    //volatile 禁止jvm进行指令重排
+    private volatile static PropertyUtil instance = null;
+    private Properties props;
+    private static final String fileName = "application.properties";
+
+    private PropertyUtil(){
+        props = new Properties();
+        InputStream in = null;
+        try {
+            in = PropertyUtil.class.getClassLoader().getResourceAsStream(fileName);
+            props.load(in);
+        } catch (Exception e){
+            logger.error("PropertyUtil init error.", e);
+        } finally {
+            try {
+                if(in != null) {
+                    in.close();
+                }
+            } catch (IOException e) {
+                logger.error("close file InputStream error.", e);
+            }
+        }
+    }
+
+    public static PropertyUtil getInstance(){
+        if(instance == null){
+            init();
+        }
+        return instance;
+    }
+
+    private synchronized static void init(){
+        if(instance == null){
+            instance = new PropertyUtil();
+        }
+    }
+
+    public String getProperty(String key){
+        if(props == null) {
+            init();
+        }
+        return props.getProperty(key);
+    }
+
+    public int getPropertyForInt(String key){
+        return Integer.parseInt(getProperty(key));
+    }
+
+    public long getPropertyForLong(String key){
+        return Long.parseLong(getProperty(key));
+    }
+
+    public boolean getPropertyForBoolean(String key){
+        return Boolean.parseBoolean(getProperty(key));
+    }
+}

src/main/resources/application.properties

+#redis config
+#common redis config
+redis.port=6379
+redis.max_active=10
+redis.max_idle=5
+redis.min_idle=1
+redis.max_wait=10000
+redis.timeout=10000
+redis.test_while_idle=90000
+redis.test_on_borrow=false
+
+hadoop.redis.host=r-bp11df1c1400afa4.redis.rds.aliyuncs.com
+hadoop.redis.auth=UjTD4apxUgu4xNVTnRAtqQt
+
+#hive config
+hive.drivername=org.apache.hive.jdbc.HiveDriver
+hive.url=jdbc:hive2://172.16.1.21:10000/default
+hive.username=hive
+hive.password=hive
\ No newline at end of file

target/classes/application.properties

+#redis config
+#common redis config
+#redis\u8BBF\u95EE\u7AEF\u53E3
+redis.port=6379
+#\u6700\u5927\u53EF\u7528\u8FDE\u63A5\u6570\uFF0C\u5982\u679C\u8D4B\u503C\u4E3A-1\uFF0C\u5219\u8868\u793A\u4E0D\u9650\u5236\uFF1B\u5982\u679Cpool\u5DF2\u7ECF\u5206\u914D\u4E86maxActive\u4E2Ajedis\u5B9E\u4F8B\uFF0C\u5219\u6B64\u65F6pool\u7684\u72B6\u6001\u4E3Aexhausted(\u8017\u5C3D)\u3002
+redis.max_active=10
+#\u63A7\u5236\u4E00\u4E2Apool\u6700\u591A\u6709\u591A\u5C11\u4E2A\u72B6\u6001\u4E3Aidle(\u7A7A\u95F2\u7684)\u7684redis\u8FDE\u63A5
+redis.max_idle=5
+#\u63A7\u5236\u4E00\u4E2Apool\u6700\u5C11\u6709\u591A\u5C11\u4E2A\u72B6\u6001\u4E3Aidle(\u7A7A\u95F2\u7684)\u7684redis\u8FDE\u63A5
+redis.min_idle=1
+#\u7B49\u5F85\u53EF\u7528\u8FDE\u63A5\u7684\u6700\u5927\u65F6\u95F4\uFF0C\u5355\u4F4D\u6BEB\u79D2\uFF0C\u9ED8\u8BA4\u503C\u4E3A-1\uFF0C\u8868\u793A\u6C38\u4E0D\u8D85\u65F6\u3002\u5982\u679C\u8D85\u8FC7\u7B49\u5F85\u65F6\u95F4\uFF0C\u5219\u76F4\u63A5\u629B\u51FAJedisConnectionException\uFF1B
+redis.max_wait=10000
+#\u8FDE\u63A5\u7684\u8D85\u65F6\u65F6\u95F4
+redis.timeout=10000
+#\u6BCF\u969490S testWhileIdle\u4E00\u6B21
+redis.test_while_idle=90000
+#\u5728borrow\u4E00\u4E2Ajedis\u5B9E\u4F8B\u65F6\uFF0C\u662F\u5426\u63D0\u524D\u8FDB\u884Cvalidate\u64CD\u4F5C\uFF1B\u5982\u679C\u4E3Atrue\uFF0C\u5219\u5F97\u5230\u7684jedis\u5B9E\u4F8B\u5747\u662F\u53EF\u7528\u7684,\u4F46\u4F1A\u5F71\u54CD\u6027\u80FD
+redis.test_on_borrow=false
+
+#hadoop redis cluster (\u963F\u91CC\u4E9116G\u96C6\u7FA4)
+hadoop.redis.host=r-bp11df1c1400afa4.redis.rds.aliyuncs.com
+#\u8BBF\u95EE\u5BC6\u7801
+hadoop.redis.auth=UjTD4apxUgu4xNVTnRAtqQt
+
+#hive config
+hive.drivername=org.apache.hive.jdbc.HiveDriver
+hive.url=jdbc:hive2://172.16.1.21:10000/default
+hive.username=hive
+hive.password=hive
\ No newline at end of file

target/maven-archiver/pom.properties

+#Generated by Maven
+#Sun Jul 15 18:13:42 CST 2018
+version=1.0-SNAPSHOT
+groupId=com.duiba.synredis
+artifactId=syn2redis

target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst

+com/duiba/synredis/util/PropertyUtil.class
+com/duiba/synredis/util/HadoopRedisUtil.class
+com/duiba/synredis/util/HiveUtil.class
+com/duiba/synredis/SynHive2Redis.class

target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst

+/Users/larkbird/Library/Mobile Documents/com~apple~CloudDocs/work/yss/project/SynRedis/src/main/java/com/duiba/synredis/SynHive2Redis.java