千亿级数仓>商品维度数据装载

本文详述了在数仓项目中,针对商品维度数据的拉链表构建过程,包括全量导入和增量导入两个阶段。在全量导入时,将2019年09月08日前的所有ODS数据导入拉链历史记录表;增量导入时,以2019年09月09日为例,介绍如何通过Kettle抽取、Spark SQL更新历史数据并合并加载到临时表,最后导入到历史拉链表中。测试环节确保数据正确性。

摘要生成于 C知道 ,由 DeepSeek-R1 满血版支持, 前往体验 >

5 数仓项目 - 商品维度数据装载

使用拉链表解决商品SCD问题

5.1 dw层建表

-- dw层建表
DROP TABLE IF EXISTS `itcast_dw`.`dim_goods`;
CREATE TABLE `itcast_dw`.`dim_goods`(
  goodsId bigint,
  goodsSn string,
  productNo string,
  goodsName string,
  goodsImg string,
  shopId bigint,
  goodsType bigint,
  marketPrice double,
  shopPrice double,
  warnStock bigint,
  goodsStock bigint,
  goodsUnit string,
  goodsTips string,
  isSale bigint,
  isBest bigint,
  isHot bigint,
  isNew bigint,
  isRecom bigint,
  goodsCatIdPath string,
  goodsCatId bigint,
  shopCatId1 bigint,
  shopCatId2 bigint,
  brandId bigint,
  goodsDesc string,
  goodsStatus bigint,
  saleNum bigint,
  saleTime string,
  visitNum bigint,
  appraiseNum bigint,
  isSpec bigint,
  gallery string,
  goodsSeoKeywords string,
  illegalRemarks string,
  dataFlag bigint,
  createTime string,
  isFreeShipping bigint,
  goodsSerachKeywords string,
  modifyTime string,
  dw_start_date string,
  dw_end_date string
)
STORED AS PARQUET;

5.2 具体步骤

拉链表设计一共分为以下几个步骤:

  • 1、第一次全量导入
    所有的ODS数据全部导入到拉链历史记录表中
  • 2、增量导入(某天,举例:2018-09-09)
    增量导入某天的数据到ODS分区
    合并历史数据
    通过连接查询方式更新

1全量导入

  • 将所有 2019年09月08日以前创建的商品以及修改的数据全部导入到拉链历史记录表中
    操作步骤:
  • 1、使用Kettle将20190908以前的数据抽取到ods
SELECT *
FROM itcast_ods.itcast_goods
WHERE DATE_FORMAT(createtime, '%Y%m%d') <= '20190908' OR DATE_FORMAT(modifyTime, '%Y%m%d') <= '20190908';
  • 2、使用spark sql将全量数据导入到dw层维度表
set spark.sql.shuffle.partitions=1; --shuffle时的分区数,默认是200个
-- 使用spark sql将全量数据导入到dw层维度表
insert overwrite table `itcast_dw`.`dim_goods`
select
  goodsId,
  goodsSn,
  productNo,
  goodsName,
  goodsImg,
  shopId,
  goodsType,
  marketPrice,
  shopPrice,
  warnStock,
  goodsStock,
  goodsUnit,
  goodsTips,
  isSale,
  isBest,
  isHot,
  isNew,
  isRecom,
  goodsCatIdPath,
  goodsCatId,
  shopCatId1,
  shopCatId2,
  brandId,
  goodsDesc,
  goodsStatus,
  saleNum,
  saleTime,
  visitNum,
  appraiseNum,
  isSpec,
  gallery,
  goodsSeoKeywords,
  illegalRemarks,
  dataFlag,
  createTime,
  isFreeShipping,
  goodsSerachKeywords,
  modifyTime,
    case when modifyTime is not null
      then from_unixtime(unix_timestamp(modifyTime, 'yyyy-MM-dd HH:mm:ss'),'yyyy-MM-dd')
      else from_unixtime(unix_timestamp(createTime, 'yyyy-MM-dd HH:mm:ss'), 'yyyy-MM-dd') 
      end as dw_start_date,
   '9999-12-31' as dw_end_date
from
  `itcast_ods`.`itcast_goods` t
where dt='20190908';

2增量导入

  • 将2019年09月09日创建的、修改的数据全部导入到历史拉链表中
    操作步骤:
  • 1、使用Kettle将20190909创建的、或者修改的数据抽取到ods
SELECT *
FROM itcast_goods
WHERE DATE_FORMAT(createtime, '%Y%m%d') = '${dt}' OR DATE_FORMAT(modifyTime, '%Y%m%d') = '${dt}';
  • 2、编写spark-sql更新历史数据
-- 更新历史数据
select
  dw.goodsId,
  dw.goodsSn,
  dw.productNo,
  dw.goodsName,
  dw.goodsImg,
  dw.shopId,
  dw.goodsType,
  dw.marketPrice,
  dw.shopPrice,
  dw.warnStock,
  dw.goodsStock,
  dw.goodsUnit,
  dw.goodsTips,
  dw.isSale,
  dw.isBest,
  dw.isHot,
  dw.isNew,
  dw.isRecom,
  dw.goodsCatIdPath,
  dw.goodsCatId,
  dw.shopCatId1,
  dw.shopCatId2,
  dw.brandId,
  dw.goodsDesc,
  dw.goodsStatus,
  dw.saleNum,
  dw.saleTime,
  dw.visitNum,
  dw.appraiseNum,
  dw.isSpec,
  dw.gallery,
  dw.goodsSeoKeywords,
  dw.illegalRemarks,
  dw.dataFlag,
  dw.createTime,
  dw.isFreeShipping,
  dw.goodsSerachKeywords,
  dw.modifyTime,
  dw.dw_start_date,
  case when dw.dw_end_date = '9999-12-31' and ods.goodsId is not null
      then '2019-09-08'
      else dw.dw_end_date
      end as dw_end_date
from
  `itcast_dw`.`dim_goods` dw
  left join 
  (select * from `itcast_ods`.`itcast_goods` where dt='20190909') ods
   on dw.goodsId = ods.goodsId;
  • 3、编写spark-sql获取当日数据
-- 今日数据
select
  goodsId,
  goodsSn,
  productNo,
  goodsName,
  goodsImg,
  shopId,
  goodsType,
  marketPrice,
  shopPrice,
  warnStock,
  goodsStock,
  goodsUnit,
  goodsTips,
  isSale,
  isBest,
  isHot,
  isNew,
  isRecom,
  goodsCatIdPath,
  goodsCatId,
  shopCatId1,
  shopCatId2,
  brandId,
  goodsDesc,
  goodsStatus,
  saleNum,
  saleTime,
  visitNum,
  appraiseNum,
  isSpec,
  gallery,
  goodsSeoKeywords,
  illegalRemarks,
  dataFlag,
  createTime,
  isFreeShipping,
  goodsSerachKeywords,
  modifyTime,
  case when modifyTime is not null
      then from_unixtime(unix_timestamp(modifyTime, 'yyyy-MM-dd HH:mm:ss'),'yyyy-MM-dd')
      else from_unixtime(unix_timestamp(createTime, 'yyyy-MM-dd HH:mm:ss'), 'yyyy-MM-dd') 
      end as dw_start_date,
   '9999-12-31' as dw_end_date
from
  `itcast_ods`.`itcast_goods`
where dt = '20190909';
  • 4、将历史数据、当日数据合并加载到临时表
-- 将历史数据、当日数据合并加载到临时表
drop table if exists `itcast_dw`.`tmp_dim_goods_history`;
create table `itcast_dw`.`tmp_dim_goods_history`
as
select
  dw.goodsId,
  dw.goodsSn,
  dw.productNo,
  dw.goodsName,
  dw.goodsImg,
  dw.shopId,
  dw.goodsType,
  dw.marketPrice,
  dw.shopPrice,
  dw.warnStock,
  dw.goodsStock,
  dw.goodsUnit,
  dw.goodsTips,
  dw.isSale,
  dw.isBest,
  dw.isHot,
  dw.isNew,
  dw.isRecom,
  dw.goodsCatIdPath,
  dw.goodsCatId,
  dw.shopCatId1,
  dw.shopCatId2,
  dw.brandId,
  dw.goodsDesc,
  dw.goodsStatus,
  dw.saleNum,
  dw.saleTime,
  dw.visitNum,
  dw.appraiseNum,
  dw.isSpec,
  dw.gallery,
  dw.goodsSeoKeywords,
  dw.illegalRemarks,
  dw.dataFlag,
  dw.createTime,
  dw.isFreeShipping,
  dw.goodsSerachKeywords,
  dw.modifyTime,
  dw.dw_start_date,
  case when dw.dw_end_date >= '2019-09-09' and ods.goodsId is not null
      then '2019-09-08'
      else dw.dw_end_date
      end as dw_end_date
from
  `itcast_dw`.`dim_goods` dw
  left join 
  (select * from `itcast_ods`.`itcast_goods` where dt='20190909') ods
   on dw.goodsId = ods.goodsId
union all
select
  goodsId,
  goodsSn,
  productNo,
  goodsName,
  goodsImg,
  shopId,
  goodsType,
  marketPrice,
  shopPrice,
  warnStock,
  goodsStock,
  goodsUnit,
  goodsTips,
  isSale,
  isBest,
  isHot,
  isNew,
  isRecom,
  goodsCatIdPath,
  goodsCatId,
  shopCatId1,
  shopCatId2,
  brandId,
  goodsDesc,
  goodsStatus,
  saleNum,
  saleTime,
  visitNum,
  appraiseNum,
  isSpec,
  gallery,
  goodsSeoKeywords,
  illegalRemarks,
  dataFlag,
  createTime,
  isFreeShipping,
  goodsSerachKeywords,
  modifyTime,
  case when modifyTime is not null
      then from_unixtime(unix_timestamp(modifyTime, 'yyyy-MM-dd HH:mm:ss'),'yyyy-MM-dd')
      else from_unixtime(unix_timestamp(createTime, 'yyyy-MM-dd HH:mm:ss'), 'yyyy-MM-dd') 
      end as dw_start_date,
   '9999-12-31' as dw_end_date
from
  `itcast_ods`.`itcast_goods`
where dt = '20190909';
  • 5、将历史数据、当日数据导入到历史拉链表
-- 将历史数据、当日数据导入到历史拉链表
insert overwrite table `itcast_dw`.`dim_goods`
select * from `itcast_dw`.`tmp_dim_goods_history`;
​
-- 获取2019-09-09日的商品数据
select * from `itcast_dw`.`dim_goods` where dw_start_date <= '2019-09-09' and dw_end_date >= '2019-09-09' limit 10;

3 测试

操作步骤:

  • 1、将mysql中的一条数据的修改日期 改为 2019-09-10
  • 2、设置kettle命名参数,重新抽取数据这一条数据到 20190910 分区
  • 3、重新执行 spark-sql脚本加载数据到临时表
-- 导入2019-09-10的历史拉链数据
-- 将历史数据、当日数据合并加载到临时表
drop table if exists `itcast_dw`.`tmp_dim_goods_history`;
create table `itcast_dw`.`tmp_dim_goods_history`
as
select
  dw.goodsId,
  dw.goodsSn,
  dw.productNo,
  dw.goodsName,
  dw.goodsImg,
  dw.shopId,
  dw.goodsType,
  dw.marketPrice,
  dw.shopPrice,
  dw.warnStock,
  dw.goodsStock,
  dw.goodsUnit,
  dw.goodsTips,
  dw.isSale,
  dw.isBest,
  dw.isHot,
  dw.isNew,
  dw.isRecom,
  dw.goodsCatIdPath,
  dw.goodsCatId,
  dw.shopCatId1,
  dw.shopCatId2,
  dw.brandId,
  dw.goodsDesc,
  dw.goodsStatus,
  dw.saleNum,
  dw.saleTime,
  dw.visitNum,
  dw.appraiseNum,
  dw.isSpec,
  dw.gallery,
  dw.goodsSeoKeywords,
  dw.illegalRemarks,
  dw.dataFlag,
  dw.createTime,
  dw.isFreeShipping,
  dw.goodsSerachKeywords,
  dw.modifyTime,
  dw.dw_start_date,
  case when dw.dw_end_date >= '2019-09-10' and ods.goodsId is not null
      then '2019-09-09'
      else dw.dw_end_date
      end as dw_end_date
from
  `itcast_dw`.`dim_goods` dw
  left join 
  (select * from `itcast_ods`.`itcast_goods` where dt='20190910') ods
   on dw.goodsId = ods.goodsId
union all
select
  goodsId,
  goodsSn,
  productNo,
  goodsName,
  goodsImg,
  shopId,
  goodsType,
  marketPrice,
  shopPrice,
  warnStock,
  goodsStock,
  goodsUnit,
  goodsTips,
  isSale,
  isBest,
  isHot,
  isNew,
  isRecom,
  goodsCatIdPath,
  goodsCatId,
  shopCatId1,
  shopCatId2,
  brandId,
  goodsDesc,
  goodsStatus,
  saleNum,
  saleTime,
  visitNum,
  appraiseNum,
  isSpec,
  gallery,
  goodsSeoKeywords,
  illegalRemarks,
  dataFlag,
  createTime,
  isFreeShipping,
  goodsSerachKeywords,
  modifyTime,
  case when modifyTime is not null
      then from_unixtime(unix_timestamp(modifyTime, 'yyyy-MM-dd HH:mm:ss'),'yyyy-MM-dd')
      else from_unixtime(unix_timestamp(createTime, 'yyyy-MM-dd HH:mm:ss'), 'yyyy-MM-dd') 
      end as dw_start_date,
   '9999-12-31' as dw_end_date
from
  `itcast_ods`.`itcast_goods`
where dt = '20190910';
  • 4、重新导入数据到历史拉链表
-- 将历史数据、当日数据导入到历史拉链表
insert overwrite table `itcast_dw`.`dim_goods`
select * from `itcast_dw`.`tmp_dim_goods_history`;
  • 5、查看对应商品id的历史拉链数据
select * from `itcast_dw`.`dim_goods` where goodsId = 100134;
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值