5 数仓项目 - 商品维度数据装载
使用拉链表解决商品SCD问题
5.1 dw层建表
-- dw层建表
DROP TABLE IF EXISTS `itcast_dw`.`dim_goods`;
CREATE TABLE `itcast_dw`.`dim_goods`(
goodsId bigint,
goodsSn string,
productNo string,
goodsName string,
goodsImg string,
shopId bigint,
goodsType bigint,
marketPrice double,
shopPrice double,
warnStock bigint,
goodsStock bigint,
goodsUnit string,
goodsTips string,
isSale bigint,
isBest bigint,
isHot bigint,
isNew bigint,
isRecom bigint,
goodsCatIdPath string,
goodsCatId bigint,
shopCatId1 bigint,
shopCatId2 bigint,
brandId bigint,
goodsDesc string,
goodsStatus bigint,
saleNum bigint,
saleTime string,
visitNum bigint,
appraiseNum bigint,
isSpec bigint,
gallery string,
goodsSeoKeywords string,
illegalRemarks string,
dataFlag bigint,
createTime string,
isFreeShipping bigint,
goodsSerachKeywords string,
modifyTime string,
dw_start_date string,
dw_end_date string
)
STORED AS PARQUET;
5.2 具体步骤
拉链表设计一共分为以下几个步骤:
- 1、第一次全量导入
所有的ODS数据全部导入到拉链历史记录表中 - 2、增量导入(某天,举例:2018-09-09)
增量导入某天的数据到ODS分区
合并历史数据
通过连接查询方式更新
1全量导入
- 将所有 2019年09月08日以前创建的商品以及修改的数据全部导入到拉链历史记录表中
操作步骤: - 1、使用Kettle将20190908以前的数据抽取到ods
SELECT *
FROM itcast_ods.itcast_goods
WHERE DATE_FORMAT(createtime, '%Y%m%d') <= '20190908' OR DATE_FORMAT(modifyTime, '%Y%m%d') <= '20190908';
- 2、使用spark sql将全量数据导入到dw层维度表
set spark.sql.shuffle.partitions=1; --shuffle时的分区数,默认是200个
-- 使用spark sql将全量数据导入到dw层维度表
insert overwrite table `itcast_dw`.`dim_goods`
select
goodsId,
goodsSn,
productNo,
goodsName,
goodsImg,
shopId,
goodsType,
marketPrice,
shopPrice,
warnStock,
goodsStock,
goodsUnit,
goodsTips,
isSale,
isBest,
isHot,
isNew,
isRecom,
goodsCatIdPath,
goodsCatId,
shopCatId1,
shopCatId2,
brandId,
goodsDesc,
goodsStatus,
saleNum,
saleTime,
visitNum,
appraiseNum,
isSpec,
gallery,
goodsSeoKeywords,
illegalRemarks,
dataFlag,
createTime,
isFreeShipping,
goodsSerachKeywords,
modifyTime,
case when modifyTime is not null
then from_unixtime(unix_timestamp(modifyTime, 'yyyy-MM-dd HH:mm:ss'),'yyyy-MM-dd')
else from_unixtime(unix_timestamp(createTime, 'yyyy-MM-dd HH:mm:ss'), 'yyyy-MM-dd')
end as dw_start_date,
'9999-12-31' as dw_end_date
from
`itcast_ods`.`itcast_goods` t
where dt='20190908';
2增量导入
- 将2019年09月09日创建的、修改的数据全部导入到历史拉链表中
操作步骤: - 1、使用Kettle将20190909创建的、或者修改的数据抽取到ods
SELECT *
FROM itcast_goods
WHERE DATE_FORMAT(createtime, '%Y%m%d') = '${dt}' OR DATE_FORMAT(modifyTime, '%Y%m%d') = '${dt}';
- 2、编写spark-sql更新历史数据
-- 更新历史数据
select
dw.goodsId,
dw.goodsSn,
dw.productNo,
dw.goodsName,
dw.goodsImg,
dw.shopId,
dw.goodsType,
dw.marketPrice,
dw.shopPrice,
dw.warnStock,
dw.goodsStock,
dw.goodsUnit,
dw.goodsTips,
dw.isSale,
dw.isBest,
dw.isHot,
dw.isNew,
dw.isRecom,
dw.goodsCatIdPath,
dw.goodsCatId,
dw.shopCatId1,
dw.shopCatId2,
dw.brandId,
dw.goodsDesc,
dw.goodsStatus,
dw.saleNum,
dw.saleTime,
dw.visitNum,
dw.appraiseNum,
dw.isSpec,
dw.gallery,
dw.goodsSeoKeywords,
dw.illegalRemarks,
dw.dataFlag,
dw.createTime,
dw.isFreeShipping,
dw.goodsSerachKeywords,
dw.modifyTime,
dw.dw_start_date,
case when dw.dw_end_date = '9999-12-31' and ods.goodsId is not null
then '2019-09-08'
else dw.dw_end_date
end as dw_end_date
from
`itcast_dw`.`dim_goods` dw
left join
(select * from `itcast_ods`.`itcast_goods` where dt='20190909') ods
on dw.goodsId = ods.goodsId;
- 3、编写spark-sql获取当日数据
-- 今日数据
select
goodsId,
goodsSn,
productNo,
goodsName,
goodsImg,
shopId,
goodsType,
marketPrice,
shopPrice,
warnStock,
goodsStock,
goodsUnit,
goodsTips,
isSale,
isBest,
isHot,
isNew,
isRecom,
goodsCatIdPath,
goodsCatId,
shopCatId1,
shopCatId2,
brandId,
goodsDesc,
goodsStatus,
saleNum,
saleTime,
visitNum,
appraiseNum,
isSpec,
gallery,
goodsSeoKeywords,
illegalRemarks,
dataFlag,
createTime,
isFreeShipping,
goodsSerachKeywords,
modifyTime,
case when modifyTime is not null
then from_unixtime(unix_timestamp(modifyTime, 'yyyy-MM-dd HH:mm:ss'),'yyyy-MM-dd')
else from_unixtime(unix_timestamp(createTime, 'yyyy-MM-dd HH:mm:ss'), 'yyyy-MM-dd')
end as dw_start_date,
'9999-12-31' as dw_end_date
from
`itcast_ods`.`itcast_goods`
where dt = '20190909';
- 4、将历史数据、当日数据合并加载到临时表
-- 将历史数据、当日数据合并加载到临时表
drop table if exists `itcast_dw`.`tmp_dim_goods_history`;
create table `itcast_dw`.`tmp_dim_goods_history`
as
select
dw.goodsId,
dw.goodsSn,
dw.productNo,
dw.goodsName,
dw.goodsImg,
dw.shopId,
dw.goodsType,
dw.marketPrice,
dw.shopPrice,
dw.warnStock,
dw.goodsStock,
dw.goodsUnit,
dw.goodsTips,
dw.isSale,
dw.isBest,
dw.isHot,
dw.isNew,
dw.isRecom,
dw.goodsCatIdPath,
dw.goodsCatId,
dw.shopCatId1,
dw.shopCatId2,
dw.brandId,
dw.goodsDesc,
dw.goodsStatus,
dw.saleNum,
dw.saleTime,
dw.visitNum,
dw.appraiseNum,
dw.isSpec,
dw.gallery,
dw.goodsSeoKeywords,
dw.illegalRemarks,
dw.dataFlag,
dw.createTime,
dw.isFreeShipping,
dw.goodsSerachKeywords,
dw.modifyTime,
dw.dw_start_date,
case when dw.dw_end_date >= '2019-09-09' and ods.goodsId is not null
then '2019-09-08'
else dw.dw_end_date
end as dw_end_date
from
`itcast_dw`.`dim_goods` dw
left join
(select * from `itcast_ods`.`itcast_goods` where dt='20190909') ods
on dw.goodsId = ods.goodsId
union all
select
goodsId,
goodsSn,
productNo,
goodsName,
goodsImg,
shopId,
goodsType,
marketPrice,
shopPrice,
warnStock,
goodsStock,
goodsUnit,
goodsTips,
isSale,
isBest,
isHot,
isNew,
isRecom,
goodsCatIdPath,
goodsCatId,
shopCatId1,
shopCatId2,
brandId,
goodsDesc,
goodsStatus,
saleNum,
saleTime,
visitNum,
appraiseNum,
isSpec,
gallery,
goodsSeoKeywords,
illegalRemarks,
dataFlag,
createTime,
isFreeShipping,
goodsSerachKeywords,
modifyTime,
case when modifyTime is not null
then from_unixtime(unix_timestamp(modifyTime, 'yyyy-MM-dd HH:mm:ss'),'yyyy-MM-dd')
else from_unixtime(unix_timestamp(createTime, 'yyyy-MM-dd HH:mm:ss'), 'yyyy-MM-dd')
end as dw_start_date,
'9999-12-31' as dw_end_date
from
`itcast_ods`.`itcast_goods`
where dt = '20190909';
- 5、将历史数据、当日数据导入到历史拉链表
-- 将历史数据、当日数据导入到历史拉链表
insert overwrite table `itcast_dw`.`dim_goods`
select * from `itcast_dw`.`tmp_dim_goods_history`;
-- 获取2019-09-09日的商品数据
select * from `itcast_dw`.`dim_goods` where dw_start_date <= '2019-09-09' and dw_end_date >= '2019-09-09' limit 10;
3 测试
操作步骤:
- 1、将mysql中的一条数据的修改日期 改为 2019-09-10
- 2、设置kettle命名参数,重新抽取数据这一条数据到 20190910 分区
- 3、重新执行 spark-sql脚本加载数据到临时表
-- 导入2019-09-10的历史拉链数据
-- 将历史数据、当日数据合并加载到临时表
drop table if exists `itcast_dw`.`tmp_dim_goods_history`;
create table `itcast_dw`.`tmp_dim_goods_history`
as
select
dw.goodsId,
dw.goodsSn,
dw.productNo,
dw.goodsName,
dw.goodsImg,
dw.shopId,
dw.goodsType,
dw.marketPrice,
dw.shopPrice,
dw.warnStock,
dw.goodsStock,
dw.goodsUnit,
dw.goodsTips,
dw.isSale,
dw.isBest,
dw.isHot,
dw.isNew,
dw.isRecom,
dw.goodsCatIdPath,
dw.goodsCatId,
dw.shopCatId1,
dw.shopCatId2,
dw.brandId,
dw.goodsDesc,
dw.goodsStatus,
dw.saleNum,
dw.saleTime,
dw.visitNum,
dw.appraiseNum,
dw.isSpec,
dw.gallery,
dw.goodsSeoKeywords,
dw.illegalRemarks,
dw.dataFlag,
dw.createTime,
dw.isFreeShipping,
dw.goodsSerachKeywords,
dw.modifyTime,
dw.dw_start_date,
case when dw.dw_end_date >= '2019-09-10' and ods.goodsId is not null
then '2019-09-09'
else dw.dw_end_date
end as dw_end_date
from
`itcast_dw`.`dim_goods` dw
left join
(select * from `itcast_ods`.`itcast_goods` where dt='20190910') ods
on dw.goodsId = ods.goodsId
union all
select
goodsId,
goodsSn,
productNo,
goodsName,
goodsImg,
shopId,
goodsType,
marketPrice,
shopPrice,
warnStock,
goodsStock,
goodsUnit,
goodsTips,
isSale,
isBest,
isHot,
isNew,
isRecom,
goodsCatIdPath,
goodsCatId,
shopCatId1,
shopCatId2,
brandId,
goodsDesc,
goodsStatus,
saleNum,
saleTime,
visitNum,
appraiseNum,
isSpec,
gallery,
goodsSeoKeywords,
illegalRemarks,
dataFlag,
createTime,
isFreeShipping,
goodsSerachKeywords,
modifyTime,
case when modifyTime is not null
then from_unixtime(unix_timestamp(modifyTime, 'yyyy-MM-dd HH:mm:ss'),'yyyy-MM-dd')
else from_unixtime(unix_timestamp(createTime, 'yyyy-MM-dd HH:mm:ss'), 'yyyy-MM-dd')
end as dw_start_date,
'9999-12-31' as dw_end_date
from
`itcast_ods`.`itcast_goods`
where dt = '20190910';
- 4、重新导入数据到历史拉链表
-- 将历史数据、当日数据导入到历史拉链表
insert overwrite table `itcast_dw`.`dim_goods`
select * from `itcast_dw`.`tmp_dim_goods_history`;
- 5、查看对应商品id的历史拉链数据
select * from `itcast_dw`.`dim_goods` where goodsId = 100134;