hive 数据导入 es

es-hadoop插件

上传es-hadoop插件到集群

准备hive数据

-- 连接hive
beeline -u "jdbc:hive2://worker-1:10000/;principal=hive/worker-1@HAINIU.COM"
-- 创建临时表
create table if not exists xiniu.hivetable(
    pk string,
    col1 int,
    col2 boolean,
    col3 timestamp,
    col4 string
)
comment 'hive表'
row format delimited fields terminated by '\t'
;
-- 加载数据
load data inpath '/eslib/testfile' into table xiniu.hivetable;

导入hive数据到es

• 上传es-hadoop jar包

hadoop fs -put /opt/elasticsearch-hadoop-7.13.1.jar /eslib/

• 加载es-hadoop jar包

add jar hdfs:///eslib/elasticsearch-hadoop-7.13.1.jar

• 创建es的hive外表

CREATE EXTERNAL TABLE xiniu.hive2es(
pk string,
col1 string,
col2 string,
col3 string,
col4 string
)STORED BY 'org.elasticsearch.hadoop.hive.EsStorageHandler'
TBLPROPERTIES('es.resource'='hivemappinges/_doc',
'es.nodes'='worker-1:9200,worker-2:9200,worker-3:9200',
'es.index.auto.create'='TRUE',
'es.index.refresh_interval' = '-1',
'es.index.number_of_replicas' = '0',
'es.batch.write.retry.count' = '6',
'es.batch.write.retry.wait' = '60s',
'es.mapping.name' = 'pk:pk,col1:col1,col2:col2,col3:col3,col4:col4'
);

• 插入数据到es的hive外表

INSERT OVERWRITE TABLE xiniu.hive2es SELECT pk,col1,col2,col3,col4 FROM xiniu.hivetable;