renyupeng vor 3 Jahren
Ursprung
Commit
b48a55ab47

+ 4 - 2
Apietl/bytedance_etl/Q4Task/AppBytedanceFileVideoGet.py

@@ -188,7 +188,8 @@ from (
                                      from (select material_id, create_time, signature,
                                                   row_number() over (partition by material_id order by create_time) as rn
                                            from media_api.bytedance_file_video_get
-                                           where material_id is not null 
+                                           where material_id is not null  and date_format(create_time, '%Y-%m-%d') >=
+                                                             date_sub('{date_time}', INTERVAL 6 DAY)
                                      )s    where rn =1) video
                                         inner join
                                     (
@@ -223,7 +224,8 @@ from (
                                      from (select material_id, create_time, signature,
                                                   row_number() over (partition by material_id order by create_time) as rn
                                            from media_api.bytedance_file_video_get
-                                           where material_id is not null 
+                                           where material_id is not null and date_format(create_time, '%Y-%m-%d') >=
+                                                             date_sub('{date_time}', INTERVAL 6 DAY)
                                      )s    where rn =1) video
                                         inner join
                                     (

+ 2 - 64
Apietl/bytedance_etl/Q4Task/AppBytedanceFileVideoGetAll.py

@@ -170,73 +170,11 @@ from (
       group by code, tag_category_id, tag_category_name, parent_id, parent_name) m
      on v.signature = m.code
          left join (
-    select stat_datetime, signature, sum
-    from (
-             select stat_datetime, signature, sum, row_number() over (partition by signature order by sum) as rn
-             from (
-                      select stat_datetime,
-                             signature,
-                             sum(cost) over (partition by signature order by stat_datetime) as sum
-                      from (
-                               select stat_datetime, video.signature, cost
-                               from (select material_id, signature, create_time
-                                     from (select material_id, create_time, signature,
-                                                  row_number() over (partition by material_id order by create_time) as rn
-                                           from media_api.bytedance_file_video_get
-                                           where material_id is not null and advertiser_id={account_id}
-
-                                     )s    where rn =1) video
-                                        inner join
-                                    (
-                                        select stat_datetime, material_id, sum(cost) as cost
-                                        from media_api.bytedance_material_report_daily
-                                        where image_mode in (
-                                                             'CREATIVE_IMAGE_MODE_VIDEO',
-                                                             'CREATIVE_IMAGE_MODE_VIDEO_VERTICAL',
-                                                             'CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO'
-                                            ) and advertiser_id={account_id}
-                                        group by stat_datetime, material_id
-                                    ) rep
-                                    on video.material_id = rep.material_id
-                               where rep.stat_datetime between date_format(video.create_time, '%Y%m%d') and date_format(date_add(video.create_time, INTERVAL 6 DAY), '%Y%m%d')) t) t
-             where sum >=5000
-         ) t
-    where rn = 1
+   select signature,effect_date as stat_datetime  from application.hot_mateiral where media_id =1 
 ) effect
                    on v.signature = effect.signature
          left join (
-    select stat_datetime, signature, sum
-    from (
-             select stat_datetime, signature, sum, row_number() over (partition by signature order by sum) as rn
-             from (
-                      select stat_datetime,
-                             signature,
-                             sum(cost) over (partition by signature order by stat_datetime) as sum
-                      from (
-                               select stat_datetime, video.signature, cost
-                               from (select material_id, signature, create_time
-                                     from (select material_id, create_time, signature,
-                                                  row_number() over (partition by material_id order by create_time) as rn
-                                           from media_api.bytedance_file_video_get 
-                                           where material_id is not null and advertiser_id={account_id}
-
-                                     )s    where rn =1) video
-                                        inner join
-                                    (
-                                        select stat_datetime, material_id, sum(cost) as cost
-                                        from media_api.bytedance_material_report_daily
-                                        where image_mode in (
-                                                             'CREATIVE_IMAGE_MODE_VIDEO',
-                                                             'CREATIVE_IMAGE_MODE_VIDEO_VERTICAL',
-                                                             'CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO'
-                                            ) and advertiser_id={account_id}
-                                        group by stat_datetime, material_id
-                                    ) rep
-                                    on video.material_id = rep.material_id
-                               where rep.stat_datetime between date_format(video.create_time, '%Y%m%d') and date_format(date_add(video.create_time, INTERVAL 13 DAY), '%Y%m%d')) t) t
-             where sum >=50000
-         ) t
-    where rn = 1
+  select signature,hot_date as stat_datetime from application.hot_mateiral where media_id =1 
 ) hot
                    on v.signature = hot.signature
 group by advertiser_id, id,  v.signature, material_id, tag_category_id, parent_id, tag_code       

+ 3 - 529
Apietl/kwai_etl/Q4Task/AppKuaishouVideoListAll.py

@@ -18,471 +18,7 @@ class AppKuaishouVideoListAll:
     def execSql(self):
         try:
             sql = """select account_id from media_api.kuaishou_video_list 
-             where account_id in( 11020929
-,10879895
-,10976614
-,10770582
-,10026522
-,10887276
-,10294287
-,10594012
-,9901147
-,10638855
-,10956498
-,11125920
-,10853283
-,11043448
-,10824501
-,10970243
-,9864376
-,10752420
-,10036809
-,10671464
-,10953994
-,10419635
-,9914592
-,11050116
-,10853258
-,9928420
-,9914528
-,10970240
-,10248813
-,10251324
-,10638878
-,9864375
-,10956523
-,10824507
-,10853318
-,10550945
-,11049251
-,10976586
-,10960823
-,11041534
-,10951696
-,10956607
-,10956337
-,11041486
-,10721442
-,10953563
-,10879680
-,11081989
-,10269620
-,10953565
-,10970233
-,11096264
-,10853255
-,10895917
-,10985484
-,10854196
-,10796789
-,10828167
-,10853263
-,10853223
-,11018826
-,11019908
-,11051812
-,11068960
-,11049245
-,10613305
-,10977846
-,11068966
-,11085925
-,10682454
-,10852008
-,10956153
-,10535876
-,10980626
-,11106992
-,10745250
-,9911489
-,11050986
-,11122738
-,10648462
-,11049231
-,10853250
-,11041531
-,10410970
-,11088993
-,11041483
-,9928555
-,11122737
-,11122779
-,11128390
-,10982457
-,10422897
-,10976622
-,11020928
-,10964778
-,11050123
-,11049249
-,11041524
-,11085941
-,10894829
-,11081621
-,11034180
-,11050984
-,10888465
-,10821254
-,10721463
-,9911539
-,11068833
-,11132894
-,11151380
-,11039159
-,11081777
-,10713401
-,11042742
-,11085954
-,10879940
-,9677344
-,10613309
-,11039141
-,11039184
-,11040711
-,10961088
-,10895994
-,10431249
-,11089023
-,10901927
-,10980636
-,10269637
-,10824508
-,11041474
-,10311495
-,11051312
-,10976582
-,10879673
-,10187441
-,10931785
-,11042694
-,10294098
-,10887021
-,10528480
-,11018824
-,10901949
-,10970249
-,10174512
-,10901969
-,10853233
-,10648397
-,11068959
-,11032325
-,10752430
-,11041535
-,10854180
-,10962093
-,11041466
-,11076757
-,10843780
-,10890118
-,10895984
-,10851805
-,10985477
-,10718730
-,10852001
-,10639008
-,10894855
-,10294227
-,10422899
-,10964769
-,11018827
-,10528485
-,11038195
-,10960785
-,10976602
-,10980622
-,11039281
-,10982381
-,11085936
-,11081755
-,9889915
-,10613310
-,10642263
-,11049258
-,10956634
-,10528422
-,11068961
-,10438264
-,10838810
-,9677284
-,10964781
-,10771833
-,10853300
-,10956221
-,11107522
-,10550475
-,11125263
-,11051315
-,10964776
-,10888450
-,10937199
-,10888460
-,11042560
-,10901753
-,10956559
-,10628429
-,10878020
-,11081801
-,11042704
-,11041533
-,10752428
-,10976619
-,11042553
-,10752416
-,9914573
-,11088989
-,11049244
-,11151532
-,10770579
-,10462547
-,10346164
-,10838846
-,10671472
-,10752439
-,11096276
-,11049899
-,11018823
-,10528412
-,11185556
-,10799930
-,10901945
-,11132864
-,11085938
-,11042718
-,10970221
-,11096860
-,10982463
-,10888454
-,11132880
-,10888489
-,11154678
-,11043293
-,11166262
-,10888430
-,11107506
-,10879938
-,10824513
-,11079104
-,10888444
-,11185593
-,11149535
-,11195182
-,10671467
-,10977924
-,10878016
-,11085923
-,10528445
-,11076845
-,11109669
-,10556372
-,10520848
-,10977892
-,11076857
-,10422901
-,11124958
-,11126402
-,11127459
-,9948710
-,11076865
-,10676395
-,10205022
-,10931809
-,11107526
-,11006740
-,10965686
-,11041496
-,11041527
-,11153735
-,10422908
-,11050127
-,11006739
-,10438266
-,10853254
-,11105485
-,9998024
-,11107529
-,10773683
-,10036933
-,10976617
-,11068965
-,10972873
-,10649214
-,11039188
-,9839558
-,10980628
-,10224952
-,10528473
-,10881687
-,10964767
-,10770580
-,11204282
-,10876815
-,10964765
-,10414252
-,10757517
-,11041490
-,10628433
-,10876835
-,11122754
-,10884086
-,10854175
-,11139968
-,11022344
-,11139974
-,10628431
-,11122775
-,11042733
-,10854173
-,11185597
-,10854176
-,10294100
-,11122772
-,11038203
-,8157345
-,11164892
-,11150075
-,11038201
-,11166578
-,11176754
-,11153732
-,11195181
-,11185648
-,11175503
-,10838859
-,11152545
-,11152373
-,11162555
-,10964768
-,11185566
-,11195186
-,11081810
-,11150077
-,11148522
-,11185647
-,10964764
-,11139991
-,11233956
-,10535855
-,11041499
-,10759492
-,11081824
-,10506987
-,10506989
-,11099359
-,10876824
-,10885098
-,11151544
-,11162586
-,11204799
-,11185576
-,11039193
-,11185653
-,11185651
-,11150089
-,10092342
-,11041507
-,11239841
-,11179741
-,11239842
-,11020922
-,10506996
-,11020921
-,11255717
-,11122769
-,11185657
-,11020927
-,11234246
-,11151515
-,11107512
-,11162572
-,11233792
-,10874786
-,10876793
-,11185582
-,11214775
-,10494982
-,11255718
-,11042563
-,10874807
-,11165128
-,11162591
-,10211147
-,10976630
-,10885136
-,11105478
-,11049904
-,11318377
-,11046422
-,11251067
-,11218224
-,11109671
-,9868203
-,10976625
-,11320623
-,11105479
-,11006742
-,11105483
-,11105477
-,11049988
-,11040581
-,11019932
-,11185604
-,11255720
-,11178424
-,10885123
-,10876801
-,11218294
-,11049911
-,11255719
-,11303575
-,11303732
-,11178434
-,10977017
-,9911624
-,11228057
-,11042566
-,11229059
-,11318358
-,10870304
-,11309634
-,10517799
-,11195190
-,10669812
-,11041519
-,11122771
-,10874792
-,9947988
-,11050451
-,10976855
-,11050975
-,11050454
-,11239843
-,10976632
-,10648480
-,11195187
-,11162573
-,10648521
-,11229058
-,10648545
-,11178443
-,11369685
-,11041522
-,10890123
-,10506997
-,10890121
-,11255721
-,10414251
-,11178437
-,11126427
-,10852006
-,11139983
-,11023397
-,10770554
-,11122758
-,11178452
-,11041513
-,11122756
-) group by account_id """
+               group by account_id """
             account_list = TidbConn.quary(self.conn, sql)
             for account_id in account_list:
                 try:
@@ -620,73 +156,11 @@ from (
       group by code, tag_category_id, tag_category_name, parent_id, parent_name) m
      on v.signature = m.code
          left join (
-    select stat_date, signature, sum
-    from (
-             select stat_date, signature, sum, row_number() over (partition by signature order by sum) as rn
-             from (
-                      select stat_date,
-                             signature,
-                             sum(cost) over (partition by signature order by stat_date) as sum
-                      from (
-                               select rep.stat_date, video.signature, cost
-                               from (select photo_id, signature, stat_date
-                                     from (select photo_id,
-                                                  stat_date,
-                                                  signature,
-                                                  row_number() over (partition by photo_id order by stat_date) rn
-                                           from media_api.kuaishou_video_list
-                                           where photo_id is not null
-                                             and photo_id != 0 and account_id ={account_id}
-                                           group by photo_id, signature) s
-                                     where rn = 1) video
-                                        left join
-                                    (
-                                        select stat_date, photo_id, sum(charge) as cost
-                                        from media_api.kuaishou_material_video_report_daily
-                                        where photo_id is not null
-                                          and photo_id != 0 and advertiser_id ={account_id}
-                                        group by stat_date, photo_id
-                                    ) rep
-                                    on video.photo_id = rep.photo_id
-                               where rep.stat_date between date_format(video.stat_date, '%Y%m%d') and date_format(date_add(video.stat_date, INTERVAL 6 DAY), '%Y%m%d')) t) t
-             where sum >= 5000
-         ) t
-    where rn = 1
+                select signature,effect_date as stat_date from application.effect_mateiral where media_id =2
 ) effect
                    on effect.signature = v.signature
          left join (
-    select stat_date, signature, sum
-    from (
-             select stat_date, signature, sum, row_number() over (partition by signature order by sum) as rn
-             from (
-                      select stat_date,
-                             signature,
-                             sum(cost) over (partition by signature order by stat_date) as sum
-                      from (
-                               select rep.stat_date, video.signature, cost
-                               from (select photo_id, signature, stat_date
-                                     from (select photo_id,
-                                                  stat_date,
-                                                  signature,
-                                                  row_number() over (partition by photo_id order by stat_date) rn
-                                           from media_api.kuaishou_video_list
-                                           where photo_id is not null 
-                                             and photo_id != 0 and account_id ={account_id}
-                                           group by photo_id, signature) s
-                                     where rn = 1) video
-                                        left join
-                                    (
-                                        select stat_date, photo_id, sum(charge) as cost
-                                        from media_api.kuaishou_material_video_report_daily
-                                        where photo_id is not null and advertiser_id ={account_id}
-                                          and photo_id != 0
-                                        group by stat_date, photo_id
-                                    ) rep
-                                    on video.photo_id = rep.photo_id
-                               where rep.stat_date between date_format(video.stat_date, '%Y%m%d') and date_format(date_add(video.stat_date, INTERVAL 13 DAY), '%Y%m%d')) t) t
-             where sum >= 50000
-         ) t
-    where rn = 1
+     select signature,hot_date as stat_date from application.hot_mateiral where media_id =2
 ) hot
                    on hot.signature = v.signature
 group by account_id, photo_id, v.stat_date, v.signature, tag_category_id, parent_id, tag_code