Skip to content

A
aliyun-elasticsearch-demo
Commit ea9bed7e authored by 武越's avatar 武越
Browse files
Options

Initial commit

parents
Show whitespace changes
Inline Side-by-side
Showing with 1155 additions and 0 deletions
.gitignore 0 → 100644
View file @ ea9bed7e
.gradle
/build/
!gradle/wrapper/gradle-wrapper.jar
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
/out/
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
\ No newline at end of file
build.gradle 0 → 100644
View file @ ea9bed7e
buildscript {
ext {
springBootVersion = '2.0.4.RELEASE'
}
repositories {
mavenCentral()
}
dependencies {
classpath("org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}")
}
}
apply plugin: 'java'
apply plugin: 'eclipse'
apply plugin: 'org.springframework.boot'
apply plugin: 'io.spring.dependency-management'
group = 'indi.wuyue'
version = '0.0.1-SNAPSHOT'
sourceCompatibility = 1.8
repositories {
mavenCentral()
maven {
url "https://artifacts.elastic.co/maven"
}
maven {
url "https://oss.sonatype.org/content/groups/public/"
}
}
dependencies {
compile('org.springframework.boot:spring-boot-starter-web')
testCompile('org.springframework.boot:spring-boot-starter-test')
compile group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.6'
compile 'org.elasticsearch.client:transport:5.5.3'
compile "org.elasticsearch.client:x-pack-transport:5.5.3"
compile 'org.elasticsearch:elasticsearch:5.5.3'
compile 'io.searchbox:jest:5.3.3'
}
gradle/wrapper/gradle-wrapper.properties 0 → 100644
View file @ ea9bed7e
#Tue Feb 06 12:27:20 CET 2018
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-4.8.1-bin.zip
gradlew 0 → 100755
View file @ ea9bed7e
#!/usr/bin/env sh
##############################################################################
##
## Gradle start up script for UN*X
##
##############################################################################
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
PRG="$0"
# Need this for relative symlinks.
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`"/$link"
fi
done
SAVED="`pwd`"
cd "`dirname \"$PRG\"`/" >/dev/null
APP_HOME="`pwd -P`"
cd "$SAVED" >/dev/null
APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS=""
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
warn ( ) {
echo "$*"
}
die ( ) {
echo
echo "$*"
echo
exit 1
}
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
nonstop=false
case "`uname`" in
CYGWIN* )
cygwin=true
;;
Darwin* )
darwin=true
;;
MINGW* )
msys=true
;;
NONSTOP* )
nonstop=true
;;
esac
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD="java"
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
MAX_FD_LIMIT=`ulimit -H -n`
if [ $? -eq 0 ] ; then
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
MAX_FD="$MAX_FD_LIMIT"
fi
ulimit -n $MAX_FD
if [ $? -ne 0 ] ; then
warn "Could not set maximum file descriptor limit: $MAX_FD"
fi
else
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
fi
fi
# For Darwin, add options to specify how the application appears in the dock
if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi
# For Cygwin, switch paths to Windows format before running java
if $cygwin ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"`
# We build the pattern for arguments to be converted via cygpath
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
SEP=""
for dir in $ROOTDIRSRAW ; do
ROOTDIRS="$ROOTDIRS$SEP$dir"
SEP="|"
done
OURCYGPATTERN="(^($ROOTDIRS))"
# Add a user-defined pattern to the cygpath arguments
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
fi
# Now convert the arguments - kludge to limit ourselves to /bin/sh
i=0
for arg in "$@" ; do
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
else
eval `echo args$i`="\"$arg\""
fi
i=$((i+1))
done
case $i in
(0) set -- ;;
(1) set -- "$args0" ;;
(2) set -- "$args0" "$args1" ;;
(3) set -- "$args0" "$args1" "$args2" ;;
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
# Escape application args
save ( ) {
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
echo " "
}
APP_ARGS=$(save "$@")
# Collect all arguments for the java command, following the shell quoting and substitution rules
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
cd "$(dirname "$0")"
fi
exec "$JAVACMD" "$@"
gradlew.bat 0 → 100644
View file @ ea9bed7e
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
set DIRNAME=%~dp0
if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS=
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if "%ERRORLEVEL%" == "0" goto init
echo.
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto init
echo.
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:init
@rem Get command-line arguments, handling Windows variants
if not "%OS%" == "Windows_NT" goto win9xME_args
:win9xME_args
@rem Slurp the command line arguments.
set CMD_LINE_ARGS=
set _SKIP=2
:win9xME_args_slurp
if "x%~1" == "x" goto execute
set CMD_LINE_ARGS=%*
:execute
@rem Setup the command line
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
:end
@rem End local scope for the variables with windows NT shell
if "%ERRORLEVEL%"=="0" goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
exit /b 1
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega
settings.gradle 0 → 100644
View file @ ea9bed7e
rootProject.name = 'elasticsearch'
src/main/java/indi/wuyue/elasticsearch/ElasticsearchApplication.java 0 → 100644
View file @ ea9bed7e
package indi.wuyue.elasticsearch;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class ElasticsearchApplication {
public static void main(String[] args) {
SpringApplication.run(ElasticsearchApplication.class, args);
}
}
src/main/java/indi/wuyue/elasticsearch/api/TestController.java 0 → 100644
View file @ ea9bed7e
package indi.wuyue.elasticsearch.api;
import indi.wuyue.elasticsearch.service.JestService;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
import java.util.List;
@RestController
public class TestController {
@Resource
private JestService service;
@RequestMapping(value = "/test/jest", method = RequestMethod.GET)
public List<String> testJest() throws Exception {
return service.test();
}
}
src/main/java/indi/wuyue/elasticsearch/basic/ClientTest.java 0 → 100644
View file @ ea9bed7e
package indi.wuyue.elasticsearch.basic;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient;
import java.net.InetAddress;
public class ClientTest {
public static void main(String[] args) throws Exception {
Settings settings = Settings.builder()
.put("xpack.security.transport.ssl.enabled", false)
.put("xpack.security.user", "elastic:1zO^U6!zPxHr9vex")
.put("client.transport.sniff", false).build();
try (PreBuiltXPackTransportClient client = new PreBuiltXPackTransportClient(settings)) {
client.addTransportAddress(new InetSocketTransportAddress(
InetAddress.getByName("es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com"), 9300));
ClusterHealthResponse healths = client.admin().cluster().prepareHealth().get();
System.out.println(healths.getClusterName());
} finally {
}
}
// public static void main(String[] args) throws Exception {
//
// Settings settings = Settings.builder()
// .put("xpack.security.user", "elastic:1zO^U6!zPxHr9vex")
// .put("http.basic.user", "elastic")
// .put("http.basic.password", "1zO^U6!zPxHr9vex")
// .build();
//
// TransportClient client = new PreBuiltTransportClient(settings)
// .addTransportAddress(new InetSocketTransportAddress(
// InetAddress.getByName("es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com"), 9300));
//
//
// CreateIndexResponse response = client.admin().indices().prepareCreate("bigdate")
// .setSettings(Settings.builder()
// .put("index.number_of_shards", 3)
// .put("index.number_of_replicas", 2))
// .addMapping("student1", "{\n" +
// "\t\"student1\": {\n" +
// "\t\t\"properties\": {\n" +
// "\t\t\t\"area\": {\n" +
// "\t\t \t\"type\": \"long\"\n" +
// "\t\t },\n" +
// "\t\t \"recordtime\": {\n" +
// "\t\t \t\"type\": \"long\"\n" +
// "\t\t },\n" +
// "\t\t \"count\": {\n" +
// "\t\t \t\"type\": \"long\"\n" +
// "\t\t },\n" +
// "\t\t \"usertype\": {\n" +
// "\t\t \t\"type\": \"long\"\n" +
// "\t\t }\n" +
// "\t\t}\n" +
// "\t}\n" +
// "}").get();
//
// // on shutdown
// client.close();
// }
}
src/main/java/indi/wuyue/elasticsearch/basic/IndexStat.java 0 → 100644
View file @ ea9bed7e
package indi.wuyue.elasticsearch.basic;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
public class IndexStat {
public static void main(String[] args) throws Exception {
// 凭据提供器
CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(
// 认证范围
new AuthScope("es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com", 9200),
// 认证用户名和密码
//账号：elastic
//密码：1zO^U6!zPxHr9vex
new UsernamePasswordCredentials("elastic", "1zO^U6!zPxHr9vex"));
CloseableHttpClient httpclient = HttpClients.custom()
.setDefaultCredentialsProvider(credsProvider)
.build();
try {
//GET请求
// HttpGet httpget = new HttpGet("http://es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200/bigdata/_stats?pretty");
HttpGet httpget = new HttpGet("http://es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200/bigdata?pretty");
System.out.println("执行请求：" + httpget.getRequestLine());
CloseableHttpResponse response = httpclient.execute(httpget);
try {
System.out.println("----------------------------------------");
System.out.println("响应：" + response.getStatusLine());
System.out.println("内容：" + EntityUtils.toString(response.getEntity()));
System.out.println("----------------------------------------");
} finally {
response.close();
}
} finally {
httpclient.close();
}
}
}
src/main/java/indi/wuyue/elasticsearch/basic/JestClientSearchClient.java 0 → 100644
View file @ ea9bed7e
package indi.wuyue.elasticsearch.basic;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.core.Search;
import io.searchbox.core.SearchResult;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
public class JestClientSearchClient {
public static void main(String[] args) throws Exception {
// Construct a new Jest client according to configuration via factory
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(new HttpClientConfig
.Builder("http://es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200")
.defaultCredentials("elastic", "1zO^U6!zPxHr9vex")
.multiThreaded(true)
.build());
JestClient client = factory.getObject();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.matchQuery("app_id", "17775"));
Search search = new Search.Builder(searchSourceBuilder.toString())
// multiple index or types can be added.
.addIndex("es_tuia_app_di_test")
.addType("2018-07-30")
.build();
SearchResult result = client.execute(search);
}
}
src/main/java/indi/wuyue/elasticsearch/basic/JestHiveMigrateClient.java 0 → 100644
View file @ ea9bed7e
package indi.wuyue.elasticsearch.basic;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.JestResult;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.indices.CreateIndex;
import io.searchbox.indices.mapping.PutMapping;
import org.elasticsearch.common.settings.Settings;
public class JestHiveMigrateClient {
private static final String mapping = "{\n" +
"\t\"2018-06-03\": {\n" +
"\t\t\"properties\": {\n" +
"\t\t\t\"act_request_count\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"advert_click_count\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"duration_type\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"participate_uv\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"finish_time\": {\n" +
"\t\t\t\t\"type\": \"string\"\n" +
"\t\t\t},\n" +
"\t\t\t\"act_click_count\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"act_suc_response_count\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"act_click_uv\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"app_name\": {\n" +
"\t\t\t\t\"type\": \"keyword\"\n" +
"\t\t\t},\n" +
"\t\t\t\"launch_count\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"start_time\": {\n" +
"\t\t\t\t\"type\": \"keyword\"\n" +
"\t\t\t},\n" +
"\t\t\t\"cur_date\": {\n" +
"\t\t\t\t\"type\": \"keyword\"\n" +
"\t\t\t},\n" +
"\t\t\t\"act_expose_count\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"act_request_uv\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"advert_request_count\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"sdk_uv\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"participate_count\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"duration_time\": {\n" +
"\t\t\t\t\"type\": \"keyword\"\n" +
"\t\t\t},\n" +
"\t\t\t\"sdk_pv\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"advert_exposure_count\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"advert_fee\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"app_id\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t}\n" +
"\t\t}\n" +
"\t}\n" +
"}";
public static void main(String[] args) throws Exception {
// Construct a new Jest client according to configuration via factory
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(new HttpClientConfig
.Builder("http://es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200")
.defaultCredentials("elastic", "1zO^U6!zPxHr9vex")
.multiThreaded(true)
.build());
JestClient client = factory.getObject();
Settings.Builder settingsBuilder = Settings.builder();
settingsBuilder.put("number_of_shards",5);
settingsBuilder.put("number_of_replicas",1);
client.execute(new CreateIndex.Builder("es_tuia_app_di")
.settings(settingsBuilder.build().getAsMap()).build());
PutMapping putMapping = new PutMapping.Builder(
"es_tuia_app_di",
"2018-07-30",
mapping)
.build();
JestResult result = client.execute(putMapping);
System.out.println("----------");
}
}
src/main/java/indi/wuyue/elasticsearch/basic/JestHiveMigrateClient2.java 0 → 100644
View file @ ea9bed7e
package indi.wuyue.elasticsearch.basic;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.JestResult;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.indices.CreateIndex;
import io.searchbox.indices.mapping.PutMapping;
import org.elasticsearch.common.settings.Settings;
public class JestHiveMigrateClient2 {
private static final String mapping = "{\n" +
" \"2018-07-30\": {\n" +
" \"properties\": {\n" +
" \"app_id\": {\n" +
" \"type\": \"long\"\n" +
" },\n" +
" \"app_name\": {\n" +
" \"type\": \"keyword\"\n" +
" }\n" +
" }\n" +
" }\n" +
"}";
public static void main(String[] args) throws Exception {
// Construct a new Jest client according to configuration via factory
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(new HttpClientConfig
.Builder("http://es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200")
.defaultCredentials("elastic", "1zO^U6!zPxHr9vex")
.multiThreaded(true)
.build());
JestClient client = factory.getObject();
Settings.Builder settingsBuilder = Settings.builder();
settingsBuilder.put("number_of_shards",5);
settingsBuilder.put("number_of_replicas",1);
client.execute(new CreateIndex.Builder("es_tuia_app_di_test")
.settings(settingsBuilder.build().getAsMap()).build());
PutMapping putMapping = new PutMapping.Builder(
"es_tuia_app_di_test",
"2018-07-30",
mapping)
.build();
JestResult result = client.execute(putMapping);
System.out.println("----------");
}
}
src/main/java/indi/wuyue/elasticsearch/basic/JestMigrateClient.java 0 → 100644
View file @ ea9bed7e
package indi.wuyue.elasticsearch.basic;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.JestResult;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.indices.CreateIndex;
import io.searchbox.indices.mapping.PutMapping;
import org.elasticsearch.common.settings.Settings;
public class JestMigrateClient {
private static final String mapping = "{\n" +
"\t\"2018-07-30\": {\n" +
"\t\t\"properties\": {\n" +
"\t\t\t\"ae\": {\n" +
"\t\t\t\t\"type\": \"keyword\"\n" +
"\t\t\t},\n" +
"\t\t\t\"consume_total\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"launch_pv\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"slot_source\": {\n" +
"\t\t\t\t\"type\": \"keyword\"\n" +
"\t\t\t},\n" +
"\t\t\t\"app_name\": {\n" +
"\t\t\t\t\"type\": \"keyword\"\n" +
"\t\t\t},\n" +
"\t\t\t\"ef_click_pv\": {\n" +
"\t\t\t\t\"type\": \"double\"\n" +
"\t\t\t},\n" +
"\t\t\t\"cur_date\": {\n" +
"\t\t\t\t\"type\": \"keyword\"\n" +
"\t\t\t},\n" +
"\t\t\t\"advert_name\": {\n" +
"\t\t\t\t\"type\": \"keyword\"\n" +
"\t\t\t},\n" +
"\t\t\t\"slot_id\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"click_pv\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"exposure_pv\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"arpu\": {\n" +
"\t\t\t\t\"type\": \"double\"\n" +
"\t\t\t},\n" +
"\t\t\t\"slot_name\": {\n" +
"\t\t\t\t\"type\": \"keyword\"\n" +
"\t\t\t},\n" +
"\t\t\t\"app_id\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"coupon_type\": {\n" +
"\t\t\t\t\"type\": \"keyword\"\n" +
"\t\t\t},\n" +
"\t\t\t\"advert_id\": {\n" +
"\t\t\t\t\"type\": \"long\"\n" +
"\t\t\t},\n" +
"\t\t\t\"ef_click_piece\": {\n" +
"\t\t\t\t\"type\": \"double\"\n" +
"\t\t\t}\n" +
"\t\t}\n" +
"\t}\n" +
"}";
public static void main(String[] args) throws Exception {
// Construct a new Jest client according to configuration via factory
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(new HttpClientConfig
.Builder("http://es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200")
.defaultCredentials("elastic", "1zO^U6!zPxHr9vex")
.multiThreaded(true)
.build());
JestClient client = factory.getObject();
Settings.Builder settingsBuilder = Settings.builder();
settingsBuilder.put("number_of_shards",5);
settingsBuilder.put("number_of_replicas",1);
client.execute(new CreateIndex.Builder("es_advert_slot_app_coupon_di")
.settings(settingsBuilder.build().getAsMap()).build());
PutMapping putMapping = new PutMapping.Builder(
"es_advert_slot_app_coupon_di",
"2018-07-30",
mapping)
.build();
JestResult result = client.execute(putMapping);
System.out.println("----------");
}
}
src/main/java/indi/wuyue/elasticsearch/basic/MigrateHttp.java 0 → 100644
View file @ ea9bed7e
package indi.wuyue.elasticsearch.basic;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
public class MigrateHttp {
public static void main(String[] args) throws Exception {
// 凭据提供器
CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(
// 认证范围
new AuthScope("es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com", 9200),
// 认证用户名和密码
//账号：elastic
//密码：1zO^U6!zPxHr9vex
new UsernamePasswordCredentials("elastic", "1zO^U6!zPxHr9vex"));
CloseableHttpClient httpclient = HttpClients.custom()
.setDefaultCredentialsProvider(credsProvider)
.build();
try {
//GET请求
HttpPut httpPut = new HttpPut("http://es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200/bigdata?pretty");
httpPut.setHeader("Content-type", "application/json");
StringEntity stringEntity = new StringEntity("{\n" +
"\t\"settings\": {\n" +
" \"number_of_shards\" : 5,\n" +
" \"number_of_replicas\" : 1\n" +
" },\n" +
"\t\"mappings\": {\n" +
"\t\t\"student1\": {\n" +
"\t\t\t\"properties\" : {\n" +
"\t\t\t\t\"area\": {\n" +
"\t\t \t\t\"type\": \"long\"\n" +
"\t\t \t},\n" +
"\t\t\t \"recordtime\": {\n" +
"\t\t\t \t\"type\": \"long\"\n" +
"\t\t\t },\n" +
"\t\t\t \"count\": {\n" +
"\t\t\t \t\"type\": \"long\"\n" +
"\t\t\t },\n" +
"\t\t\t \"usertype\": {\n" +
"\t\t\t \t\"type\": \"long\"\n" +
"\t\t\t }\n" +
"\t\t\t}\n" +
"\t\t\t\n" +
"\t\t}\n" +
"\t}\n" +
"}", "UTF-8");
httpPut.setEntity(stringEntity);
System.out.println("执行请求：" + httpPut.getRequestLine());
CloseableHttpResponse response = httpclient.execute(httpPut);
try {
System.out.println("----------------------------------------");
System.out.println("响应：" + response.getStatusLine());
System.out.println("内容：" + EntityUtils.toString(response.getEntity()));
System.out.println("----------------------------------------");
} finally {
response.close();
}
} finally {
httpclient.close();
}
}
}
src/main/java/indi/wuyue/elasticsearch/basic/TestHttp.java 0 → 100644
View file @ ea9bed7e
package indi.wuyue.elasticsearch.basic;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
/**
*
* http 9200 通过
*
*/
public class TestHttp {
public static void main(String[] args) throws Exception {
// 凭据提供器
CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(
// 认证范围
new AuthScope("es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com", 9200),
// 认证用户名和密码
//账号：elastic
//密码：1zO^U6!zPxHr9vex
new UsernamePasswordCredentials("elastic", "1zO^U6!zPxHr9vex"));
CloseableHttpClient httpclient = HttpClients.custom()
.setDefaultCredentialsProvider(credsProvider)
.build();
try {
//GET请求
HttpGet httpget = new HttpGet("http://es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200/_count?pretty");
System.out.println("执行请求：" + httpget.getRequestLine());
CloseableHttpResponse response = httpclient.execute(httpget);
try {
System.out.println("----------------------------------------");
System.out.println("响应：" + response.getStatusLine());
System.out.println("内容：" + EntityUtils.toString(response.getEntity()));
System.out.println("----------------------------------------");
} finally {
response.close();
}
} finally {
httpclient.close();
}
}
}
src/main/java/indi/wuyue/elasticsearch/basic/migrate.txt 0 → 100644
View file @ ea9bed7e
1.测试read旧集群input
input {
elasticsearch {
hosts => "172.16.1.121:9200"
index => "bigdata"
query => '{ "query": { "query_string": { "query": "*" } } }'
size => 500
scroll => "5m"
docinfo => true
}
}
output {
stdout {}
}
2.测试写入新集群output
input {
elasticsearch {
hosts => "172.16.1.121:9200"
index => "bigdata"
query => '{ "query": { "query_string": { "query": "*" } } }'
size => 500
scroll => "5m"
docinfo => true
}
}
output {
elasticsearch {
hosts => "es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200"
index => "bigdata"
user => "elastic"
password => "1zO^U6!zPxHr9vex"
}
}
2.1失败信息
[2018-07-30T11:51:53,393][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>404, :action=>["index", {:_id=>nil, :_index=>"bigdata", :_type=>"doc", :_routing=>nil}, #<LogStash::Event:0x233b403f>], :response=>{"index"=>{"_index"=>"bigdata", "_type"=>"doc", "_id"=>nil, "status"=>404, "error"=>{"type"=>"index_not_found_exception", "reason"=>"no such index and [action.auto_create_index] contains [-*] which forbids automatic creation of the index", "index_uuid"=>"_na_", "index"=>"bigdata"}}}}
原因：新集群没创建索引，并且新集群禁止写入时自动创建索引
创建索引
es官方不建议使用transport client，建议使用rest client
https://www.elastic.co/guide/en/elasticsearch/reference/5.5/setup-xpack-client.html
2.2写入新集群时发现type没有使用老集群的type
解决output增加一行
document_type => "%{[@metadata][_type]}"
input {
elasticsearch {
hosts => "172.16.1.121:9200"
index => "bigdata"
query => '{ "query": { "query_string": { "query": "*" } } }'
size => 500
scroll => "5m"
docinfo => true
}
}
output {
elasticsearch {
hosts => "es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200"
index => "bigdata"
document_type => "%{[@metadata][_type]}"
user => "elastic"
password => "1zO^U6!zPxHr9vex"
}
}
3.测试大批量数据迁移
旧集群索引es_advert_slot_app_coupon_di
JestMigrateClient
{
"2018-07-30": {
"properties": {
"ae": {
"type": "keyword"
},
"consume_total": {
"type": "long"
},
"launch_pv": {
"type": "long"
},
"slot_source": {
"type": "keyword"
},
"app_name": {
"type": "keyword"
},
"ef_click_pv": {
"type": "double"
},
"cur_date": {
"type": "keyword"
},
"advert_name": {
"type": "keyword"
},
"slot_id": {
"type": "long"
},
"click_pv": {
"type": "long"
},
"exposure_pv": {
"type": "long"
},
"arpu": {
"type": "double"
},
"slot_name": {
"type": "keyword"
},
"app_id": {
"type": "long"
},
"coupon_type": {
"type": "keyword"
},
"advert_id": {
"type": "long"
},
"ef_click_piece": {
"type": "double"
}
}
}
}
input {
elasticsearch {
hosts => "172.16.1.121:9200"
index => "es_advert_slot_app_coupon_di"
query => '{
"query": {
"match": {
"_type": "2018-07-30"
}
}
}'
docinfo => true
}
}
output {
elasticsearch {
hosts => "es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200"
index => "es_advert_slot_app_coupon_di"
document_type => "%{[@metadata][_type]}"
user => "elastic"
password => "1zO^U6!zPxHr9vex"
}
}
3.es-sql
3.1site-server未启动成功
3.2插件功能验证
验证查询，通过
http://es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200/_sql?sql=select * from es_advert_slot_app_coupon_di limit 10
dsl转换，通过
http://es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200/_sql/_explain?sql=select * from es_advert_slot_app_coupon_di limit 10
4.es-hadoop
4.1jar依赖问题
4.2脚本properties指定
TBLPROPERTIES(
'es.nodes'='es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200',
'es.nodes.wan.only'='true'
'es.net.http.auth.user'='elastic',
'es.net.http.auth.pass'='1zO^U6!zPxHr9vex',
'es.resource' = 'es_tuia_app_di_test/2018-07-30'
);
'es.nodes.wan.only'='true' 云ES服务的话这个属性一定要加
'es.net.http.auth.user'='elastic' 用户名
'es.net.http.auth.pass'='1zO^U6!zPxHr9vex' 密码
src/main/java/indi/wuyue/elasticsearch/config/JestConfig.java 0 → 100644
View file @ ea9bed7e
package indi.wuyue.elasticsearch.config;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.config.HttpClientConfig;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Scope;
@Configuration
public class JestConfig {
@Bean
@Scope("singleton")
public JestClient client() {
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(new HttpClientConfig
.Builder("http://es-cn-4590plkku0004vufk.elasticsearch.aliyuncs.com:9200")
.defaultCredentials("elastic", "1zO^U6!zPxHr9vex")
.multiThreaded(true)
.build());
JestClient client = factory.getObject();
return client;
}
}
src/main/java/indi/wuyue/elasticsearch/service/JestService.java 0 → 100644
View file @ ea9bed7e
package indi.wuyue.elasticsearch.service;
import io.searchbox.client.JestClient;
import io.searchbox.core.Search;
import io.searchbox.core.SearchResult;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.List;
@Service
public class JestService {
@Resource
private JestClient client;
public List<String> test() throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.matchQuery("app_id", "17775"));
Search search = new Search.Builder(searchSourceBuilder.toString())
// multiple index or types can be added.
.addIndex("es_tuia_app_di_test")
.addType("2018-07-30")
.build();
SearchResult result = client.execute(search);
List<String> sourceList = result.getSourceAsStringList();
return sourceList;
}
}
src/test/java/indi/wuyue/elasticsearch/ElasticsearchApplicationTests.java 0 → 100644
View file @ ea9bed7e
package indi.wuyue.elasticsearch;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringRunner.class)
@SpringBootTest
public class ElasticsearchApplicationTests {
@Test
public void contextLoads() {
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment