Can I trigger a job based on another job? [duplicate] - firebase

Question

I have some queries that run hourly and export the data from BigQuery to GCS. The daily tables seem to be updated over the next 2 days. Has anyone found a way to get the delta rows between what was exported to GCS and what was inserted in BigQuery?
This is what I am currently running (which ignores the missing data) as a scheduled script
DECLARE tables ARRAY <STRING>;
CREATE TABLE IF NOT EXISTS `project.analytics_xxx.daily_export_log`
(
table_name STRING,
insert_date TIMESTAMP
);
SET tables = (SELECT
ARRAY_AGG(TABLE_NAME) TABLES
FROM
`project.analytics_xxx.INFORMATION_SCHEMA.TABLES`
WHERE
REGEXP_CONTAINS(TABLE_NAME, 'events_\\d{8}') AND
TABLE_NAME NOT IN (SELECT TABLE_NAME FROM `project.analytics_xxx.daily_export_log`)
);
FOR tab IN
(SELECT * FROM UNNEST(tables))
DO
EXECUTE IMMEDIATE '''
EXPORT DATA
OPTIONS ( uri = CONCAT('gs://bucket/live/', format_timestamp('%Y/%m/%d/', current_timestamp()), ''' || "'" || tab.f0_ || "'" || ''', '/*_', format_timestamp('%Y%m%d%H%M%S', current_timestamp()), '.json.gz'),
format='JSON',
compression='GZIP',
overwrite=FALSE ) AS
SELECT * FROM `project.analytics_xxx.''' || tab.f0_ || '''` ''';
EXECUTE IMMEDIATE '''
INSERT INTO `project.analytics_xxx.daily_export_log` SELECT ''' || "'" || tab.f0_ || "'" || ''' table_name, current_timestamp() insert_date
''';
END FOR
I tried using event_timestamp + event_server_timestamp_offset as a cutoff measure but that failed as event_timestamp is device synced so if users have altered their system time then their timestamps would be way off.

Answer 1

I've managed to fix the issue by appending a gcs_export_timestamp column to every export table.
DECLARE tables ARRAY <STRING>;
/************************************************************************************************************
* DAILY EVENTS EXPORT *
************************************************************************************************************/
CREATE TABLE IF NOT EXISTS `project.analytics_xxx.daily_export_log`
(
table_name STRING,
insert_date TIMESTAMP
)
OPTIONS
(
expiration_timestamp=TIMESTAMP "3000-01-01"
);
SET tables = (SELECT
ARRAY_AGG(TABLE_NAME) TABLES
FROM
`project.analytics_xxx.INFORMATION_SCHEMA.TABLES`
WHERE
REGEXP_CONTAINS(TABLE_NAME, 'events_\\d{8}') AND
TABLE_NAME NOT IN (SELECT TABLE_NAME FROM `project.analytics_xxx.daily_export_log`)
);
FOR tab IN
(SELECT * FROM UNNEST(tables))
DO
EXECUTE IMMEDIATE '''
EXPORT DATA
OPTIONS (
uri = CONCAT('gs://bucket/live/', format_timestamp('%Y/%m/%d/', current_timestamp()), ''' || "'" || tab.f0_ || "'" || ''', '/*_', format_timestamp('%Y%m%d%H%M%S', current_timestamp()), '.json.gz'),
format='JSON',
compression='GZIP',
overwrite=FALSE
) AS
SELECT * FROM `project.analytics_xxx.''' || tab.f0_ || '''`
''';
EXECUTE IMMEDIATE '''
INSERT INTO `project.analytics_xxx.daily_export_log` SELECT ''' || "'" || tab.f0_ || "'" || ''' table_name, current_timestamp() insert_date
''';
EXECUTE IMMEDIATE '''
ALTER TABLE `project.analytics_xxx.''' || tab.f0_ || '''`
ADD COLUMN IF NOT EXISTS gcs_export_timestamp TIMESTAMP
''';
EXECUTE IMMEDIATE '''
UPDATE `project.analytics_xxx.''' || tab.f0_ || '''` SET
gcs_export_timestamp = current_timestamp()
WHERE gcs_export_timestamp IS NULL
''';
END FOR;
/************************************************************************************************************
* DELAYED EVENTS EXPORT *
************************************************************************************************************/
SET tables = (SELECT
ARRAY_AGG(TABLE_NAME) TABLES
FROM
`project.analytics_xxx.INFORMATION_SCHEMA.TABLES`
WHERE
REGEXP_CONTAINS(TABLE_NAME, 'events_\\d{8}')
);
FOR tab IN
(SELECT * FROM UNNEST(tables))
DO
/************ CHECK IF DAILY TABLE WAS OVERWRITTEN AND RE-ADD THE gcs_export_timestamp COLUMN ************/
BEGIN
IF (SELECT 1 FROM `region-us`.INFORMATION_SCHEMA.COLUMNS WHERE table_name = tab.f0_ AND column_name = 'gcs_export_timestamp') IS NULL THEN
IF (SELECT 1 FROM `project.analytics_xxx.daily_export_log` WHERE table_name = tab.f0_) IS NOT NULL THEN
EXECUTE IMMEDIATE '''
ALTER TABLE `project.analytics_xxx.''' || tab.f0_ || '''`
ADD COLUMN IF NOT EXISTS gcs_export_timestamp TIMESTAMP
''';
EXECUTE IMMEDIATE '''
UPDATE `project.analytics_xxx.''' || tab.f0_ || '''` SET
gcs_export_timestamp = (SELECT insert_date FROM `project.analytics_xxx.daily_export_log` WHERE table_name = \'''' || tab.f0_ || '''\')
WHERE gcs_export_timestamp IS NULL
''';
END IF;
END IF;
END;
/***************************** EXPORT DATA WHERE gcs_export_timestamp IS NULL ****************************/
BEGIN
EXECUTE IMMEDIATE '''
CREATE OR REPLACE TEMP TABLE _SESSION.tmp AS
SELECT * FROM `project.analytics_xxx.''' || tab.f0_ || '''` WHERE gcs_export_timestamp IS NULL
''';
IF (SELECT COUNT(*) cnt FROM _SESSION.tmp) > 0 THEN
EXECUTE IMMEDIATE '''
EXPORT DATA
OPTIONS (
uri = CONCAT('gs://bucket/delayed/', format_timestamp('%Y/%m/%d/', current_timestamp()), ''' || "'" || tab.f0_ || "'" || ''', '/*_', format_timestamp('%Y%m%d%H%M%S', current_timestamp()), '.json.gz'),
format='JSON',
compression='GZIP',
overwrite=FALSE
) AS
SELECT * FROM _SESSION.tmp
''';
EXECUTE IMMEDIATE '''
UPDATE `project.analytics_xxx.''' || tab.f0_ || '''` SET
gcs_export_timestamp = current_timestamp()
WHERE gcs_export_timestamp IS NULL
''';
END IF;
EXCEPTION WHEN ERROR THEN END;
END FOR;