{
"datasets": [
{
"name": "5e3eb18f",
"displayName": "Event Times",
"queryLines": [
"SELECT event_time AS event_timestamp\n",
"FROM system.lakeflow_pipeline_events_preview.pipeline_events\n",
"WHERE (:pipeline_id = 'All' OR pipeline_id = :pipeline_id)\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "82c4e3b5",
"displayName": "Pipelines Runs Errors",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
"),\n",
"runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id),\n",
"ev AS (SELECT *, coalesce(try_element_at(error.exceptions, 1).error_class, try_element_at(error.exceptions, 1).sql_state) AS ec FROM system.lakeflow_pipeline_events_preview.pipeline_events WHERE level = 'ERROR')\n",
"SELECT p.pipeline_name AS `Pipeline Name`,\n",
" p.pipeline_link AS `Pipeline Link`,\n",
" e.update_id AS `Pipeline Run Id`,\n",
" concat('', e.update_id, '') AS `Pipeline Run`,\n",
" e.origin.dataset_name AS `Target Table`,\n",
" r.create_time AS `Pipeline Run Start Time`,\n",
" e.event_time AS `Error Time`,\n",
" e.message AS `Error Message in Log`,\n",
" nullif(concat_ws('; ', transform(coalesce(e.error.exceptions, array()), x -> coalesce(x.error_class, x.sql_state))), '') AS `Error Message`,\n",
" (CASE WHEN e.ec IS NULL THEN '' WHEN e.ec = '' THEN '' ELSE e.ec END) AS `Error Code`,\n",
" to_json(e.error) AS `Error Details`\n",
"FROM ev e\n",
" LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
" LEFT JOIN runs r ON r.pipeline_id = e.pipeline_id AND r.update_id = e.update_id\n",
"WHERE (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
" AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
"ORDER BY e.event_time DESC\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "c91619f8",
"displayName": "Pipelines Metrics Freshness",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
")\n",
"SELECT pipeline_link AS `Pipeline`,\n",
" latest_event_timestamp AS `Observability data freshness`,\n",
" current_timestamp() AS `Check time`,\n",
" timestampdiff(MINUTE, latest_event_timestamp, current_timestamp()) AS `Latency at check time (minutes)`\n",
"FROM (\n",
" SELECT e.pipeline_id, p.pipeline_link, p.tags_array, max(e.event_time) AS latest_event_timestamp\n",
" FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p USING (pipeline_id)\n",
" WHERE (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id) AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
" GROUP BY 1, 2, 3)\n",
"ORDER BY latest_event_timestamp\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "a189e440",
"displayName": "Pipelines Metrics Freshness Summary",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
")\n",
"SELECT current_timestamp() AS `Observability data check time`,\n",
" min_latest AS `Oldest observability data freshness`,\n",
" max_latest AS `Latest observability data freshness`,\n",
" timestampdiff(MINUTE, min_latest, current_timestamp()) AS `Largest latency (minutes)`,\n",
" timestampdiff(MINUTE, max_latest, current_timestamp()) AS `Smallest latency (minutes)`\n",
"FROM (\n",
" SELECT min(latest_event_timestamp) AS min_latest, max(latest_event_timestamp) AS max_latest\n",
" FROM (\n",
" SELECT e.pipeline_id, max(e.event_time) AS latest_event_timestamp\n",
" FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p USING (pipeline_id)\n",
" WHERE (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id) AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
" GROUP BY 1))\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "6740610b",
"displayName": "Pipeline Ids",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
")\n",
"SELECT pipeline_id\n",
"FROM pipes\n",
"WHERE (:tag_value = '')\n",
" OR (:tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0))\n",
" OR array_contains(tags_array, :tag_value)\n"
],
"parameters": [
{
"displayName": "tag_value",
"keyword": "tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "19d796af",
"displayName": "Pipelines Runs Status",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
"),\n",
"runs AS (\n",
" SELECT pipeline_id, update_id AS pipeline_run_id,\n",
" MIN(period_start_time) AS create_time,\n",
" MAX(CASE WHEN result_state IS NOT NULL THEN period_end_time END) AS end_time,\n",
" MAX(result_state) AS latest_state\n",
" FROM system.lakeflow.pipeline_update_timeline\n",
" GROUP BY pipeline_id, update_id),\n",
"phases AS (\n",
" SELECT pipeline_id, update_id,\n",
" min(CASE WHEN st = 'INITIALIZING' THEN event_time END) AS init_start,\n",
" min(CASE WHEN st = 'RUNNING' THEN event_time END) AS run_start\n",
" FROM (SELECT pipeline_id, update_id, event_time, upper(details:update_progress.state::string) AS st\n",
" FROM system.lakeflow_pipeline_events_preview.pipeline_events WHERE event_type = 'update_progress')\n",
" GROUP BY 1, 2),\n",
"errs AS (\n",
" SELECT pipeline_id, update_id,\n",
" max_by(message, event_time) AS err_log,\n",
" max_by(nullif(concat_ws('; ', transform(coalesce(error.exceptions, array()), x -> coalesce(x.error_class, x.sql_state))), ''), event_time) AS err_msg,\n",
" max_by(coalesce(try_element_at(error.exceptions, 1).error_class, try_element_at(error.exceptions, 1).sql_state), event_time) AS err_code,\n",
" max_by(to_json(error), event_time) AS err_full,\n",
" max(event_time) AS err_time\n",
" FROM system.lakeflow_pipeline_events_preview.pipeline_events\n",
" WHERE level = 'ERROR'\n",
" GROUP BY 1, 2\n",
")\n",
"SELECT p.pipeline_link AS `Pipeline`,\n",
" p.pipeline_name AS `Pipeline Name`,\n",
" concat('', r.pipeline_run_id, '') AS `Pipeline run`,\n",
" r.create_time AS `Create time`,\n",
" r.end_time AS `End time`,\n",
" r.latest_state AS `Status String`,\n",
" concat('', coalesce(r.latest_state, '-'), '') AS `Status`,\n",
" ifnull(er.err_log, '') AS `Latest error log message`,\n",
" ifnull(er.err_msg, '') AS `Latest error message`,\n",
" ifnull(er.err_code, '') AS `Latest error code`,\n",
" er.err_full AS `Latest full error`,\n",
" timestampdiff(second, r.create_time, ifnull(r.end_time, current_timestamp())) AS `Total seconds`,\n",
" timestampdiff(second, r.create_time, ph.init_start) AS `Starting seconds`,\n",
" timestampdiff(second, ph.init_start, ph.run_start) AS `Initialization seconds`,\n",
" timestampdiff(second, ph.run_start, ifnull(r.end_time, current_timestamp())) AS `Running seconds`\n",
"FROM runs r\n",
" LEFT JOIN pipes p USING (pipeline_id)\n",
" LEFT JOIN phases ph ON ph.pipeline_id = r.pipeline_id AND ph.update_id = r.pipeline_run_id\n",
" LEFT JOIN errs er ON er.pipeline_id = r.pipeline_id AND er.update_id = r.pipeline_run_id\n",
"WHERE (:pipeline_id = 'All' OR r.pipeline_id = :pipeline_id)\n",
" AND ((r.create_time >= :event_range.min AND r.create_time <= :event_range.max)\n",
" OR (r.end_time >= :event_range.min AND r.end_time <= :event_range.max))\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "5ef35e45",
"displayName": "Pipeline Tags",
"queryLines": [
"SELECT '' AS tag_value\n",
"UNION ALL\n",
"SELECT DISTINCT tv AS tag_value\n",
"FROM (\n",
" SELECT explode(transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END))) AS tv\n",
" FROM (SELECT tags FROM system.lakeflow.pipelines WHERE tags IS NOT NULL AND cardinality(tags) > 0\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1))\n"
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "8897fc81",
"displayName": "Table Warnings",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
"),\n",
"runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id)\n",
"SELECT p.pipeline_name AS `Pipeline Name`,\n",
" p.pipeline_link AS `Pipeline Link`,\n",
" e.update_id AS `Pipeline Run Id`,\n",
" concat('', e.update_id, '') AS `Pipeline Run`,\n",
" e.origin.dataset_name AS `Target Table`,\n",
" r.create_time AS `Pipeline Start Time`,\n",
" e.event_time AS `Warning Time`,\n",
" e.message AS `Warning Log Message`\n",
"FROM system.lakeflow_pipeline_events_preview.pipeline_events e\n",
" LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
" LEFT JOIN runs r ON r.pipeline_id = e.pipeline_id AND r.update_id = e.update_id\n",
"WHERE e.level IN ('WARN', 'WARNING')\n",
" AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
" AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
" AND (:table_name = 'All' OR e.origin.dataset_name = :table_name)\n",
" AND e.origin.dataset_name IS NOT NULL\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
"ORDER BY e.event_time DESC\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "table_name",
"keyword": "table_name",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "2faba057",
"displayName": "Pipelines Hourly Error Rate",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
")\n",
"SELECT e.pipeline_id AS `Pipeline Id`,\n",
" p.pipeline_name AS `Pipeline Name`,\n",
" p.pipeline_link AS `Pipeline Link`,\n",
" date_trunc('HOUR', e.event_time) AS `Hour`,\n",
" count(*) AS `Number of Errors`\n",
"FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
"WHERE e.level = 'ERROR'\n",
" AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
" AND (date_trunc('HOUR', e.event_time) >= :event_range.min AND date_trunc('HOUR', e.event_time) <= :event_range.max)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
"GROUP BY 1, 2, 3, 4\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "2660d018",
"displayName": "Table Changes",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
"),\n",
"runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id)\n",
"SELECT p.pipeline_name AS `Pipeline Name`,\n",
" p.pipeline_link AS `Pipeline Link`,\n",
" e.update_id AS `Pipeline Run Id`,\n",
" concat('', e.update_id, '') AS `Pipeline Run`,\n",
" e.origin.flow_name AS `Flow`,\n",
" e.event_time AS `Event Time`,\n",
" ifnull(TRY_CAST(e.details:flow_progress.metrics.num_output_rows AS BIGINT), 0) AS `Appended Rows`,\n",
" ifnull(TRY_CAST(e.details:flow_progress.metrics.num_upserted_rows AS BIGINT), 0) AS `Upserted Rows`,\n",
" ifnull(TRY_CAST(e.details:flow_progress.metrics.num_deleted_rows AS BIGINT), 0) AS `Deleted Rows`,\n",
" ifnull(TRY_CAST(e.details:flow_progress.metrics.num_output_bytes AS BIGINT), 0) AS `Appended Bytes`,\n",
" ifnull(TRY_CAST(e.details:flow_progress.metrics.num_output_rows AS BIGINT), 0)\n",
" + ifnull(TRY_CAST(e.details:flow_progress.metrics.num_upserted_rows AS BIGINT), 0)\n",
" + ifnull(TRY_CAST(e.details:flow_progress.metrics.num_deleted_rows AS BIGINT), 0) AS `Total Changes`\n",
"FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
"WHERE e.event_type = 'flow_progress'\n",
" AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
" AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
" AND (e.details:flow_progress.metrics.num_output_rows IS NOT NULL\n",
" OR e.details:flow_progress.metrics.num_upserted_rows IS NOT NULL\n",
" OR e.details:flow_progress.metrics.num_deleted_rows IS NOT NULL)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "table_name",
"keyword": "table_name",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "248b74c6",
"displayName": "Table Status",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
"),\n",
"runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time, MAX(CASE WHEN result_state IS NOT NULL THEN period_end_time END) AS end_time, MAX(result_state) AS result_state FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id),\n",
"fp AS (\n",
" SELECT pipeline_id, update_id, origin.dataset_name AS dataset_name, event_time,\n",
" details:flow_progress.status::string AS status\n",
" FROM system.lakeflow_pipeline_events_preview.pipeline_events\n",
" WHERE event_type = 'flow_progress' AND origin.dataset_name IS NOT NULL),\n",
"latest_tbl AS (\n",
" SELECT pipeline_id, dataset_name,\n",
" max_by(update_id, event_time) AS update_id,\n",
" max_by(status, event_time) AS status,\n",
" max(event_time) AS updated_at\n",
" FROM fp GROUP BY 1, 2),\n",
"latest_change AS (\n",
" SELECT pipeline_id, origin.dataset_name AS dataset_name, max(event_time) AS changes_time\n",
" FROM system.lakeflow_pipeline_events_preview.pipeline_events\n",
" WHERE event_type = 'flow_progress' AND origin.dataset_name IS NOT NULL\n",
" AND TRY_CAST(details:flow_progress.metrics.num_output_rows AS BIGINT) > 0\n",
" GROUP BY 1, 2),\n",
"tbl_err AS (\n",
" SELECT pipeline_id, origin.dataset_name AS dataset_name,\n",
" max_by(update_id, event_time) AS update_id,\n",
" max_by(message, event_time) AS err_log,\n",
" max_by(nullif(concat_ws('; ', transform(coalesce(error.exceptions, array()), x -> coalesce(x.error_class, x.sql_state))), ''), event_time) AS err_msg,\n",
" max_by(coalesce(try_element_at(error.exceptions, 1).error_class, try_element_at(error.exceptions, 1).sql_state), event_time) AS err_code,\n",
" max(event_time) AS err_time\n",
" FROM system.lakeflow_pipeline_events_preview.pipeline_events WHERE level = 'ERROR' AND origin.dataset_name IS NOT NULL\n",
" GROUP BY 1, 2)\n",
"SELECT p.pipeline_name AS `Pipeline Name`,\n",
" p.pipeline_link AS `Pipeline Link`,\n",
" lt.dataset_name AS `Target Table`,\n",
" concat('', lt.update_id, '') AS `Latest Run`,\n",
" concat('', coalesce(lt.status, '-'), '') AS `Latest State`,\n",
" lt.status AS `State`,\n",
" CAST(NULL AS STRING) AS `Latest Schema`,\n",
" current_timestamp() AS `Last Data Refresh Time`,\n",
" lc.changes_time AS `Latest Changes Time`,\n",
" timestampdiff(SECOND, lc.changes_time, current_timestamp()) AS `Seconds Since Latest Change`,\n",
" te.err_time AS `Time of Latest Error`,\n",
" te.err_code AS `Code of Latest Error`,\n",
" te.err_log AS `Log Message of Latest Error`,\n",
" te.err_msg AS `Message of Latest Error`,\n",
" concat('', te.update_id, '') AS `Latest Pipeline Run with Error`,\n",
" lt.updated_at AS `Last Updated`,\n",
" timestampdiff(SECOND, lt.updated_at, current_timestamp()) AS `Seconds Since Latest Status Update`\n",
"FROM latest_tbl lt\n",
" LEFT JOIN pipes p ON p.pipeline_id = lt.pipeline_id\n",
" LEFT JOIN latest_change lc ON lc.pipeline_id = lt.pipeline_id AND lc.dataset_name = lt.dataset_name\n",
" LEFT JOIN tbl_err te ON te.pipeline_id = lt.pipeline_id AND te.dataset_name = lt.dataset_name\n",
"WHERE (:pipeline_id = 'All' OR lt.pipeline_id = :pipeline_id)\n",
" AND (lt.updated_at >= :event_range.min AND lt.updated_at <= :event_range.max)\n",
" AND (:table_name = 'All' OR lt.dataset_name = :table_name)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
],
"parameters": [
{
"displayName": "table_name",
"keyword": "table_name",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "0ac0bb93",
"displayName": "Selected Pipelines",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
")\n",
"SELECT pipeline_id AS `Pipeline Id`,\n",
" pipeline_link AS `Pipeline Name`,\n",
" pipeline_type AS `Pipeline Type`,\n",
" tags_map AS `Pipeline Tags`\n",
"FROM pipes\n",
"WHERE (:pipeline_id = 'All' OR pipeline_id = :pipeline_id)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "7103e200",
"displayName": "Table Status Per Pipeline Run",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
"),\n",
"runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time, MAX(CASE WHEN result_state IS NOT NULL THEN period_end_time END) AS end_time, MAX(result_state) AS result_state FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id),\n",
"fp AS (\n",
" SELECT pipeline_id, update_id, origin.dataset_name AS dataset_name, event_time,\n",
" details:flow_progress.status::string AS status\n",
" FROM system.lakeflow_pipeline_events_preview.pipeline_events\n",
" WHERE event_type = 'flow_progress' AND origin.dataset_name IS NOT NULL),\n",
"latest_tbl AS (\n",
" SELECT pipeline_id, dataset_name,\n",
" max_by(update_id, event_time) AS update_id,\n",
" max_by(status, event_time) AS status,\n",
" max(event_time) AS updated_at\n",
" FROM fp GROUP BY 1, 2),\n",
"latest_change AS (\n",
" SELECT pipeline_id, origin.dataset_name AS dataset_name, max(event_time) AS changes_time\n",
" FROM system.lakeflow_pipeline_events_preview.pipeline_events\n",
" WHERE event_type = 'flow_progress' AND origin.dataset_name IS NOT NULL\n",
" AND TRY_CAST(details:flow_progress.metrics.num_output_rows AS BIGINT) > 0\n",
" GROUP BY 1, 2),\n",
"tbl_err AS (\n",
" SELECT pipeline_id, origin.dataset_name AS dataset_name,\n",
" max_by(update_id, event_time) AS update_id,\n",
" max_by(message, event_time) AS err_log,\n",
" max_by(nullif(concat_ws('; ', transform(coalesce(error.exceptions, array()), x -> coalesce(x.error_class, x.sql_state))), ''), event_time) AS err_msg,\n",
" max_by(coalesce(try_element_at(error.exceptions, 1).error_class, try_element_at(error.exceptions, 1).sql_state), event_time) AS err_code,\n",
" max(event_time) AS err_time\n",
" FROM system.lakeflow_pipeline_events_preview.pipeline_events WHERE level = 'ERROR' AND origin.dataset_name IS NOT NULL\n",
" GROUP BY 1, 2),\n",
"fp_run AS (\n",
" SELECT pipeline_id, update_id, dataset_name,\n",
" max_by(status, event_time) AS status, max(event_time) AS updated_at\n",
" FROM fp GROUP BY 1, 2, 3)\n",
"SELECT p.pipeline_name AS `Pipeline Name`,\n",
" p.pipeline_link AS `Pipeline Link`,\n",
" fr.dataset_name AS `Target Table`,\n",
" concat('', fr.update_id, '') AS `Pipeline Run`,\n",
" concat('', coalesce(fr.status, '-'), '') AS `Table State`,\n",
" ifnull(te.err_code, '') AS `Error Code`,\n",
" te.err_time AS `Error Time`,\n",
" ifnull(te.err_msg, '') AS `Error Message`,\n",
" ifnull(te.err_log, '') AS `Error Message in Log`,\n",
" CAST(NULL AS STRING) AS `Latest Schema`,\n",
" rn.create_time AS `Pipeline Run Create Time`,\n",
" rn.end_time AS `Pipeline Run End Time`,\n",
" concat('', coalesce(rn.result_state, '-'), '') AS `Pipeline Run State`,\n",
" fr.updated_at AS `Last Updated`\n",
"FROM fp_run fr\n",
" LEFT JOIN pipes p ON p.pipeline_id = fr.pipeline_id\n",
" LEFT JOIN runs rn ON rn.pipeline_id = fr.pipeline_id AND rn.update_id = fr.update_id\n",
" LEFT JOIN tbl_err te ON te.pipeline_id = fr.pipeline_id AND te.dataset_name = fr.dataset_name AND te.update_id = fr.update_id\n",
"WHERE (:pipeline_id = 'All' OR fr.pipeline_id = :pipeline_id)\n",
" AND (fr.updated_at >= :event_range.min AND fr.updated_at <= :event_range.max)\n",
" AND (:table_name = 'All' OR fr.dataset_name = :table_name)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "table_name",
"keyword": "table_name",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "c0688ffc",
"displayName": "Pipelines Runs Warnings",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
"),\n",
"runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id)\n",
"SELECT p.pipeline_name AS `Pipeline Name`,\n",
" p.pipeline_link AS `Pipeline Link`,\n",
" e.update_id AS `Pipeline Run Id`,\n",
" concat('', e.update_id, '') AS `Pipeline Run`,\n",
" r.create_time AS `Pipeline Run Start Time`,\n",
" e.event_time AS `Warning Time`,\n",
" e.message AS `Warning Message`\n",
"FROM system.lakeflow_pipeline_events_preview.pipeline_events e\n",
" LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
" LEFT JOIN runs r ON r.pipeline_id = e.pipeline_id AND r.update_id = e.update_id\n",
"WHERE e.level IN ('WARN', 'WARNING')\n",
" AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
" AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
"ORDER BY e.event_time DESC\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "8db826d7",
"displayName": "Table Names",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
")\n",
"SELECT DISTINCT e.origin.dataset_name AS table_name\n",
"FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
"WHERE e.origin.dataset_name IS NOT NULL\n",
" AND e.origin.dataset_name NOT LIKE '%.cdc_staging_table'\n",
" AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "b18ee129",
"displayName": "Selected Tables",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
"),\n",
"tabs AS (SELECT DISTINCT pipeline_id, origin.dataset_name AS table_name FROM system.lakeflow_pipeline_events_preview.pipeline_events WHERE origin.dataset_name IS NOT NULL)\n",
"SELECT p.pipeline_link AS `Pipeline`,\n",
" t.table_name AS `Target Table`\n",
"FROM tabs t LEFT JOIN pipes p ON p.pipeline_id = t.pipeline_id\n",
"WHERE (:pipeline_id = 'All' OR t.pipeline_id = :pipeline_id)\n",
" AND (:table_name = 'All' OR t.table_name = :table_name)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "table_name",
"keyword": "table_name",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "3b8b7d86",
"displayName": "Table Errors",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
"),\n",
"runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id),\n",
"ev AS (SELECT *, coalesce(try_element_at(error.exceptions, 1).error_class, try_element_at(error.exceptions, 1).sql_state) AS ec FROM system.lakeflow_pipeline_events_preview.pipeline_events WHERE level = 'ERROR')\n",
"SELECT p.pipeline_name AS `Pipeline Name`,\n",
" p.pipeline_link AS `Pipeline Link`,\n",
" e.update_id AS `Pipeline Run Id`,\n",
" concat('', e.update_id, '') AS `Pipeline Run`,\n",
" e.origin.dataset_name AS `Target Table`,\n",
" r.create_time AS `Pipeline Start Time`,\n",
" e.event_time AS `Error Time`,\n",
" e.message AS `Error Log Message`,\n",
" nullif(concat_ws('; ', transform(coalesce(e.error.exceptions, array()), x -> coalesce(x.error_class, x.sql_state))), '') AS `Error Message`,\n",
" (CASE WHEN e.ec IS NULL THEN '' WHEN e.ec = '' THEN '' ELSE e.ec END) AS `Error Code`,\n",
" to_json(e.error) AS `Error Details`\n",
"FROM ev e\n",
" LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
" LEFT JOIN runs r ON r.pipeline_id = e.pipeline_id AND r.update_id = e.update_id\n",
"WHERE (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
" AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
" AND (:table_name = 'All' OR e.origin.dataset_name = :table_name)\n",
" AND e.origin.dataset_name IS NOT NULL\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
"ORDER BY e.event_time DESC\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "table_name",
"keyword": "table_name",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "62d03d54",
"displayName": "Pipelines Status",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
"),\n",
"runs AS (\n",
" SELECT pipeline_id, update_id, create_time, end_time, result_state,\n",
" row_number() OVER (PARTITION BY pipeline_id ORDER BY create_time DESC) AS rn\n",
" FROM (\n",
" SELECT pipeline_id, update_id,\n",
" MIN(period_start_time) AS create_time,\n",
" MAX(CASE WHEN result_state IS NOT NULL THEN period_end_time END) AS end_time,\n",
" MAX(result_state) AS result_state\n",
" FROM system.lakeflow.pipeline_update_timeline\n",
" GROUP BY pipeline_id, update_id)),\n",
"cnts AS (\n",
" SELECT pipeline_id, update_id,\n",
" sum(CASE WHEN level = 'ERROR' THEN 1 ELSE 0 END) AS n_err,\n",
" sum(CASE WHEN level IN ('WARN', 'WARNING') THEN 1 ELSE 0 END) AS n_warn\n",
" FROM system.lakeflow_pipeline_events_preview.pipeline_events GROUP BY 1, 2),\n",
"errs AS (\n",
" SELECT pipeline_id, update_id,\n",
" max_by(message, event_time) AS err_log,\n",
" max_by(nullif(concat_ws('; ', transform(coalesce(error.exceptions, array()), x -> coalesce(x.error_class, x.sql_state))), ''), event_time) AS err_msg,\n",
" max_by(coalesce(try_element_at(error.exceptions, 1).error_class, try_element_at(error.exceptions, 1).sql_state), event_time) AS err_code,\n",
" max_by(to_json(error), event_time) AS err_full,\n",
" max(event_time) AS err_time\n",
" FROM system.lakeflow_pipeline_events_preview.pipeline_events\n",
" WHERE level = 'ERROR'\n",
" GROUP BY 1, 2\n",
")\n",
"SELECT p.pipeline_link AS `Pipeline`,\n",
" p.pipeline_name AS `Pipeline Name`,\n",
" concat('', r.update_id, '') AS `Latest Pipeline Run`,\n",
" r.create_time AS `Latest Create time`,\n",
" r.end_time AS `Latest End time`,\n",
" r.result_state AS `Status`,\n",
" concat('', coalesce(r.result_state, '-'), '') AS `Latest Status`,\n",
" ifnull(er.err_log, '') AS `Latest error log message`,\n",
" ifnull(er.err_msg, '') AS `Latest error message`,\n",
" ifnull(er.err_code, '') AS `Latest error code`,\n",
" ifnull(c.n_err, 0) AS `Latest run errors`,\n",
" ifnull(c.n_warn, 0) AS `Latest run warnings`,\n",
" timestampdiff(second, r.create_time, ifnull(r.end_time, current_timestamp())) AS `Latest Total Seconds`,\n",
" r.end_time AS `Last Updated`\n",
"FROM runs r\n",
" LEFT JOIN pipes p USING (pipeline_id)\n",
" LEFT JOIN cnts c ON c.pipeline_id = r.pipeline_id AND c.update_id = r.update_id\n",
" LEFT JOIN errs er ON er.pipeline_id = r.pipeline_id AND er.update_id = r.update_id\n",
"WHERE r.rn = 1\n",
" AND (:pipeline_id = 'All' OR r.pipeline_id = :pipeline_id)\n",
" AND ((r.create_time >= :event_range.min AND r.create_time <= :event_range.max)\n",
" OR (r.end_time >= :event_range.min AND r.end_time <= :event_range.max))\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "b72dbc41",
"displayName": "Table Backlog",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
")\n",
"SELECT p.pipeline_name AS `Pipeline Name`,\n",
" p.pipeline_link AS `Pipeline Link`,\n",
" e.update_id AS `Pipeline Run Id`,\n",
" concat('', e.update_id, '') AS `Pipeline Run`,\n",
" e.origin.dataset_name AS `Target Table`,\n",
" e.event_time AS `Event Time`,\n",
" ifnull(TRY_CAST(e.details:flow_progress.metrics.backlog_bytes AS BIGINT), 0) AS `Backlog Bytes`,\n",
" ifnull(TRY_CAST(e.details:flow_progress.metrics.backlog_records AS BIGINT), 0) AS `Backlog Rows`,\n",
" ifnull(TRY_CAST(e.details:flow_progress.metrics.backlog_files AS BIGINT), 0) AS `Backlog Files`,\n",
" ifnull(TRY_CAST(e.details:flow_progress.metrics.backlog_seconds AS BIGINT), 0) AS `Backlog Seconds`\n",
"FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
"WHERE e.event_type = 'flow_progress'\n",
" AND e.origin.dataset_name IS NOT NULL\n",
" AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
" AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
" AND (:table_name = 'All' OR e.origin.dataset_name = :table_name)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "table_name",
"keyword": "table_name",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "e9270b22",
"displayName": "Table Expectation Checks",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
"),\n",
"runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id),\n",
"exploded AS (\n",
" SELECT e.pipeline_id, e.update_id, COALESCE(ex.dataset, e.origin.dataset_name) AS dataset_name, e.event_time,\n",
" ex.name AS expectation_name, ex.passed_records, ex.failed_records\n",
" FROM system.lakeflow_pipeline_events_preview.pipeline_events e\n",
" LATERAL VIEW explode(CAST(e.details:flow_progress.data_quality.expectations\n",
" AS ARRAY>)) AS ex\n",
" WHERE e.event_type = 'flow_progress')\n",
"SELECT p.pipeline_name AS `Pipeline Name`,\n",
" p.pipeline_link AS `Pipeline Link`,\n",
" ex.update_id AS `Pipeline Run Id`,\n",
" concat('', ex.update_id, '') AS `Pipeline Run`,\n",
" ex.dataset_name AS `Target Table`,\n",
" r.create_time AS `Pipeline Start Time`,\n",
" ex.expectation_name AS `Expectation Name`,\n",
" ex.passed_records AS `Rows Passed`,\n",
" ex.failed_records AS `Rows Failed`,\n",
" round(100.0 * ex.failed_records / NULLIF(ex.passed_records + ex.failed_records, 0), 2) AS `Failure Percentage`,\n",
" ex.event_time AS `Check Time`\n",
"FROM exploded ex\n",
" LEFT JOIN pipes p ON p.pipeline_id = ex.pipeline_id\n",
" LEFT JOIN runs r ON r.pipeline_id = ex.pipeline_id AND r.update_id = ex.update_id\n",
"WHERE ex.dataset_name IS NOT NULL\n",
" AND (:pipeline_id = 'All' OR ex.pipeline_id = :pipeline_id)\n",
" AND (ex.event_time >= :event_range.min AND ex.event_time <= :event_range.max)\n",
" AND (:table_name = 'All' OR ex.dataset_name = :table_name)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
"ORDER BY ex.event_time DESC\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "table_name",
"keyword": "table_name",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "f786fd7e",
"displayName": "Table Expectation Failures",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
"),\n",
"runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id),\n",
"exploded AS (\n",
" SELECT e.pipeline_id, e.update_id, COALESCE(ex.dataset, e.origin.dataset_name) AS dataset_name, e.event_time,\n",
" ex.name AS expectation_name, ex.passed_records, ex.failed_records\n",
" FROM system.lakeflow_pipeline_events_preview.pipeline_events e\n",
" LATERAL VIEW explode(CAST(e.details:flow_progress.data_quality.expectations\n",
" AS ARRAY>)) AS ex\n",
" WHERE e.event_type = 'flow_progress')\n",
"SELECT p.pipeline_name AS `Pipeline Name`,\n",
" p.pipeline_link AS `Pipeline Link`,\n",
" ex.update_id AS `Pipeline Run Id`,\n",
" concat('', ex.update_id, '') AS `Pipeline Run`,\n",
" ex.dataset_name AS `Target Table`,\n",
" r.create_time AS `Pipeline Start Time`,\n",
" ex.expectation_name AS `Expectation Name`,\n",
" ex.passed_records AS `Rows Passed`,\n",
" ex.failed_records AS `Rows Failed`,\n",
" round(100.0 * ex.failed_records / NULLIF(ex.passed_records + ex.failed_records, 0), 2) AS `Failure Percentage`,\n",
" ex.event_time AS `Check Time`\n",
"FROM exploded ex\n",
" LEFT JOIN pipes p ON p.pipeline_id = ex.pipeline_id\n",
" LEFT JOIN runs r ON r.pipeline_id = ex.pipeline_id AND r.update_id = ex.update_id\n",
"WHERE ex.dataset_name IS NOT NULL\n",
" AND (:pipeline_id = 'All' OR ex.pipeline_id = :pipeline_id)\n",
" AND (ex.event_time >= :event_range.min AND ex.event_time <= :event_range.max)\n",
" AND (:table_name = 'All' OR ex.dataset_name = :table_name)\n",
" AND ex.failed_records > 0\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
"ORDER BY ex.event_time DESC\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "table_name",
"keyword": "table_name",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "bc1ad993",
"displayName": "Table Expectations Dropped Rows",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
")\n",
"SELECT p.pipeline_name AS `Pipeline Name`,\n",
" p.pipeline_link AS `Pipeline Link`,\n",
" e.update_id AS `Pipeline Run Id`,\n",
" concat('', e.update_id, '') AS `Pipeline Run`,\n",
" e.origin.dataset_name AS `Target Table`,\n",
" e.event_time AS `Event Time`,\n",
" ifnull(TRY_CAST(e.details:flow_progress.data_quality.dropped_records AS BIGINT), 0) AS `Expectations Dropped Rows`\n",
"FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
"WHERE e.event_type = 'flow_progress'\n",
" AND e.origin.dataset_name IS NOT NULL\n",
" AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
" AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
" AND (:table_name = 'All' OR e.origin.dataset_name = :table_name)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
"ORDER BY e.event_time DESC\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "table_name",
"keyword": "table_name",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
],
"catalog": "main",
"schema": "databricks_ingestion_monitoring"
},
{
"name": "flow_outcomes",
"displayName": "Flow outcomes",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
")\n",
"SELECT e.details:flow_progress.status::string AS status,\n",
" approx_count_distinct(coalesce(e.update_id, '') || '|' || coalesce(e.origin.flow_id, e.origin.flow_name, '')) AS flow_runs\n",
"FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
"WHERE e.event_type = 'flow_progress'\n",
" AND e.details:flow_progress.status::string IN ('COMPLETED', 'FAILED', 'SKIPPED', 'STOPPED', 'EXCLUDED')\n",
" AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
" AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
"GROUP BY 1 ORDER BY flow_runs DESC\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
]
},
{
"name": "hygiene",
"displayName": "Config hygiene",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
")\n",
"SELECT e.event_type || ': ' || COALESCE(e.details:deprecation.feature::string, e.details:unsupported_operation.operation::string, e.details:advisory.subject::string, '-') AS item,\n",
" count(*) AS events\n",
"FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
"WHERE e.event_type IN ('deprecation', 'unsupported_operation', 'advisory')\n",
" AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
" AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
"GROUP BY 1 ORDER BY events DESC LIMIT 12\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
}
]
},
{
"name": "dd_util",
"displayName": "Cluster utilization",
"queryLines": [
"WITH pipes AS (\n",
" SELECT pipeline_id,\n",
" name AS pipeline_name,\n",
" concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
" pipeline_type,\n",
" CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
" ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
" tags AS tags_map\n",
" FROM system.lakeflow.pipelines\n",
" QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
")\n",
"SELECT CASE WHEN :param_util_grain = 'hour' THEN date_trunc('HOUR', e.event_time) ELSE date_trunc('MINUTE', e.event_time) END AS ts,\n",
" CASE WHEN :param_util_groupby = 'per cluster (pod)' THEN COALESCE(e.origin.cluster_id, '(unknown)') ELSE 'all clusters' END AS cluster,\n",
" avg(TRY_CAST(e.details:cluster_resources.task_slot_metrics.avg_task_slot_utilization AS DOUBLE)) * 100 AS slot_utilization_pct\n",
"FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
"WHERE e.event_type = 'cluster_resources'\n",
" AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
" AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
" AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
"GROUP BY 1, 2 ORDER BY ts\n"
],
"parameters": [
{
"displayName": "pipeline_id",
"keyword": "pipeline_id",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "All"
}
]
}
}
},
{
"displayName": "event_range",
"keyword": "event_range",
"dataType": "DATETIME",
"complexType": "RANGE",
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-7d/d"
},
"max": {
"value": "now/d"
}
}
}
},
{
"displayName": "pipeline_tag_value",
"keyword": "pipeline_tag_value",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": ""
}
]
}
}
},
{
"displayName": "Utilization grain",
"keyword": "param_util_grain",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "hour"
}
]
}
}
},
{
"displayName": "Cluster grouping",
"keyword": "param_util_groupby",
"dataType": "STRING",
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "all clusters"
}
]
}
}
}
]
},
{
"name": "util_grain",
"displayName": "Utilization grain options",
"queryLines": ["SELECT grain FROM VALUES ('minute', 0), ('hour', 1) AS t(grain, ord) ORDER BY ord\n"]
},
{
"name": "util_groupby",
"displayName": "Cluster grouping options",
"queryLines": [
"SELECT opt FROM VALUES ('all clusters', 0), ('per cluster (pod)', 1) AS t(opt, ord) ORDER BY ord\n"
]
}
],
"pages": [
{
"name": "cd63ec52",
"displayName": "Filtered",
"layout": [
{
"widget": {
"name": "df5934b3",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "0ac0bb93",
"fields": [
{
"name": "Pipeline Id",
"expression": "`Pipeline Id`"
},
{
"name": "Pipeline Name",
"expression": "`Pipeline Name`"
},
{
"name": "Pipeline Type",
"expression": "`Pipeline Type`"
},
{
"name": "Pipeline Tags",
"expression": "`Pipeline Tags`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 1,
"widgetType": "table",
"encodings": {
"columns": [
{
"fieldName": "Pipeline Id",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 100000,
"title": "Pipeline Id",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Pipeline Name",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 100001,
"title": "Pipeline Name",
"allowSearch": true,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 400
},
{
"fieldName": "Pipeline Type",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 100002,
"title": "Pipeline Type",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Pipeline Tags",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "complex",
"displayAs": "json",
"visible": true,
"order": 100003,
"title": "Pipeline Tags",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
]
},
"invisibleColumns": [],
"allowHTMLByDefault": false,
"itemsPerPage": 25,
"paginationSize": "default",
"condensed": true,
"withRowNumber": false,
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Selected Pipelines",
"description": "Pipelines selected using the global filters"
}
}
},
"position": {
"x": 0,
"y": 6,
"width": 6,
"height": 5
}
},
{
"widget": {
"name": "665329d1",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "b18ee129",
"fields": [
{
"name": "Pipeline",
"expression": "`Pipeline`"
},
{
"name": "Target Table",
"expression": "`Target Table`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 1,
"widgetType": "table",
"encodings": {
"columns": [
{
"fieldName": "Pipeline",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 100000,
"title": "Pipeline",
"allowSearch": true,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 400
},
{
"fieldName": "Target Table",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 100001,
"title": "Target Table",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
]
},
"invisibleColumns": [],
"allowHTMLByDefault": false,
"itemsPerPage": 25,
"paginationSize": "default",
"condensed": true,
"withRowNumber": false,
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Selected Tables",
"description": "Tables selected using the global filters"
}
}
},
"position": {
"x": 0,
"y": 11,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "1fcb6a9c",
"multilineTextboxSpec": {
"lines": [
"***Pipelines, pipeline runs and tables selected using the Global filters***"
]
}
},
"position": {
"x": 0,
"y": 4,
"width": 6,
"height": 2
}
},
{
"widget": {
"name": "5693a89a",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "19d796af",
"fields": [
{
"name": "Pipeline",
"expression": "`Pipeline`"
},
{
"name": "Pipeline run",
"expression": "`Pipeline run`"
},
{
"name": "Status",
"expression": "`Status`"
},
{
"name": "Create time",
"expression": "`Create time`"
},
{
"name": "End time",
"expression": "`End time`"
},
{
"name": "Latest error log message",
"expression": "`Latest error log message`"
},
{
"name": "Latest error message",
"expression": "`Latest error message`"
},
{
"name": "Total seconds",
"expression": "`Total seconds`"
},
{
"name": "Starting seconds",
"expression": "`Starting seconds`"
},
{
"name": "Initialization seconds",
"expression": "`Initialization seconds`"
},
{
"name": "Running seconds",
"expression": "`Running seconds`"
},
{
"name": "Latest full error",
"expression": "`Latest full error`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 1,
"widgetType": "table",
"encodings": {
"columns": [
{
"fieldName": "Pipeline",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 0,
"title": "Pipeline",
"allowSearch": true,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 350
},
{
"fieldName": "Pipeline run",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 1,
"title": "Pipeline run",
"allowSearch": false,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 350
},
{
"fieldName": "Status",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 2,
"title": "Status",
"allowSearch": false,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 150
},
{
"fieldName": "Create time",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 3,
"title": "Create time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "End time",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 4,
"title": "End time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Latest error log message",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 5,
"title": "Latest error log message",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Latest error message",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 6,
"title": "Latest error message",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Total seconds",
"numberFormat": "0",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "integer",
"displayAs": "number",
"visible": true,
"order": 7,
"title": "Total seconds",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Starting seconds",
"numberFormat": "0",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "integer",
"displayAs": "number",
"visible": true,
"order": 8,
"title": "Starting seconds",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Initialization seconds",
"numberFormat": "0",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "integer",
"displayAs": "number",
"visible": true,
"order": 10,
"title": "Initialization seconds",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Running seconds",
"numberFormat": "0",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "integer",
"displayAs": "number",
"visible": true,
"order": 11,
"title": "Running seconds",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Latest full error",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "complex",
"displayAs": "json",
"visible": true,
"order": 12,
"title": "Latest full error",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
]
},
"invisibleColumns": [
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Name",
"type": "string",
"displayAs": "string",
"order": 100001,
"title": "Pipeline Name",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Status String",
"type": "string",
"displayAs": "string",
"order": 100005,
"title": "Status String",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Latest error code",
"type": "string",
"displayAs": "string",
"order": 100009,
"title": "Latest error code",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
],
"allowHTMLByDefault": false,
"itemsPerPage": 25,
"paginationSize": "default",
"condensed": true,
"withRowNumber": false,
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Selected pipeline runs",
"description": "Pipeline runs selected using the global filters"
}
}
},
"position": {
"x": 0,
"y": 17,
"width": 6,
"height": 9
}
},
{
"widget": {
"name": "observability-data-updated",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "a189e440",
"fields": [
{
"name": "minutely(Observability data check time)",
"expression": "DATE_TRUNC(\"MINUTE\", `Observability data check time`)"
}
],
"disaggregated": true,
"orders": [
{
"direction": "DESC",
"expression": "DATE_TRUNC(\"MINUTE\", `Observability data check time`)"
}
]
}
}
],
"spec": {
"version": 2,
"widgetType": "counter",
"encodings": {
"period": {
"fieldName": "minutely(Observability data check time)"
}
},
"frame": {
"showTitle": true,
"title": "Observability Data Last Updated At"
}
}
},
"position": {
"x": 0,
"y": 0,
"width": 2,
"height": 2
}
},
{
"widget": {
"name": "oldest-observability-data-latency-minutes",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "a189e440",
"fields": [
{
"name": "Largest latency (minutes)",
"expression": "`Largest latency (minutes)`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 2,
"widgetType": "counter",
"encodings": {
"value": {
"fieldName": "Largest latency (minutes)"
}
},
"frame": {
"showTitle": true,
"title": "Oldest Observability Data Latency (in minutes)",
"showDescription": true,
"description": "See \"Observability Data Freshness\" in \"Pipelines Health\" tab for more info"
}
}
},
"position": {
"x": 2,
"y": 0,
"width": 2,
"height": 2
}
},
{
"widget": {
"name": "newest-observability-data-latency-minutes",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "a189e440",
"fields": [
{
"name": "Smallest latency (minutes)",
"expression": "`Smallest latency (minutes)`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 2,
"widgetType": "counter",
"encodings": {
"value": {
"fieldName": "Smallest latency (minutes)"
}
},
"frame": {
"showTitle": true,
"title": "Newest Observability Data Latency (in minutes)",
"showDescription": true,
"description": "See \"Observability Data Freshness\" in \"Pipelines Health\" tab for more info"
}
}
},
"position": {
"x": 4,
"y": 0,
"width": 2,
"height": 2
}
},
{
"widget": {
"name": "selected-pipelines",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "0ac0bb93",
"fields": [
{
"name": "count(*)",
"expression": "COUNT(`*`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 2,
"widgetType": "counter",
"encodings": {
"value": {
"fieldName": "count(*)"
}
},
"frame": {
"showTitle": true,
"title": "Number of Selected Pipelines",
"showDescription": true,
"description": "See \"Selected Pipelines\" below"
}
}
},
"position": {
"x": 0,
"y": 2,
"width": 2,
"height": 2
}
},
{
"widget": {
"name": "number-selected-tables",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "b18ee129",
"fields": [
{
"name": "count(*)",
"expression": "COUNT(`*`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 2,
"widgetType": "counter",
"encodings": {
"value": {
"fieldName": "count(*)"
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Number of Selected Tables",
"description": "See \"Selected Tables\" below"
}
}
},
"position": {
"x": 2,
"y": 2,
"width": 2,
"height": 2
}
},
{
"widget": {
"name": "number-selected-pipeline-runs",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "19d796af",
"fields": [
{
"name": "count(*)",
"expression": "COUNT(`*`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 2,
"widgetType": "counter",
"encodings": {
"value": {
"fieldName": "count(*)"
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Number of Selected Pipeline Runs",
"description": "See \"Selected Pipeline Runs\" below"
}
}
},
"position": {
"x": 4,
"y": 2,
"width": 2,
"height": 2
}
}
],
"pageType": "PAGE_TYPE_CANVAS"
},
{
"name": "a44ce042",
"displayName": "Pipelines Health",
"layout": [
{
"widget": {
"name": "dd476651",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "c91619f8",
"fields": [
{
"name": "Pipeline",
"expression": "`Pipeline`"
},
{
"name": "Observability data freshness",
"expression": "`Observability data freshness`"
},
{
"name": "Check time",
"expression": "`Check time`"
},
{
"name": "Latency at check time (minutes)",
"expression": "`Latency at check time (minutes)`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 1,
"widgetType": "table",
"encodings": {
"columns": [
{
"fieldName": "Pipeline",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 100000,
"title": "Pipeline",
"allowSearch": true,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 350
},
{
"fieldName": "Observability data freshness",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 100001,
"title": "Observability data freshness",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Check time",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 100002,
"title": "Check time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Latency at check time (minutes)",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 100003,
"title": "Latency at check time (minutes)",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
]
},
"invisibleColumns": [],
"allowHTMLByDefault": false,
"itemsPerPage": 25,
"paginationSize": "default",
"condensed": true,
"withRowNumber": false,
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Observability Data Freshness",
"description": "Time and latency of the latest processed event by pipeline (oldest to newest)"
}
}
},
"position": {
"x": 0,
"y": 24,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "6820f9d5",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "82c4e3b5",
"fields": [
{
"name": "Pipeline Link",
"expression": "`Pipeline Link`"
},
{
"name": "Pipeline Run",
"expression": "`Pipeline Run`"
},
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "Error Time",
"expression": "`Error Time`"
},
{
"name": "Error Code",
"expression": "`Error Code`"
},
{
"name": "Error Message in Log",
"expression": "`Error Message in Log`"
},
{
"name": "Error Message",
"expression": "`Error Message`"
},
{
"name": "Error Details",
"expression": "`Error Details`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 1,
"widgetType": "table",
"encodings": {
"columns": [
{
"fieldName": "Pipeline Link",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 1,
"title": "Pipeline Link",
"allowSearch": true,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 350
},
{
"fieldName": "Pipeline Run",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 3,
"title": "Pipeline Run",
"allowSearch": false,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 300
},
{
"fieldName": "Target Table",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 4,
"title": "Target Table",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Error Time",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 6,
"title": "Error Time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Error Code",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 7,
"title": "Error Code",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Error Message in Log",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 8,
"title": "Error Message in Log",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Error Message",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 9,
"title": "Error Message",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Error Details",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "complex",
"displayAs": "json",
"visible": true,
"order": 10,
"title": "Error Details",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
]
},
"invisibleColumns": [
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Name",
"type": "string",
"displayAs": "string",
"order": 0,
"title": "Pipeline Name",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Run Id",
"type": "string",
"displayAs": "string",
"order": 2,
"title": "Pipeline Run Id",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Run Start Time",
"type": "datetime",
"displayAs": "datetime",
"order": 5,
"title": "Pipeline Run Start Time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
],
"allowHTMLByDefault": false,
"itemsPerPage": 25,
"paginationSize": "default",
"condensed": true,
"withRowNumber": false,
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Errors in Pipelines ",
"description": "Detected errors in the pipeline runs"
}
}
},
"position": {
"x": 0,
"y": 12,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "9ac0f134",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "c0688ffc",
"fields": [
{
"name": "Pipeline Link",
"expression": "`Pipeline Link`"
},
{
"name": "Pipeline Run Id",
"expression": "`Pipeline Run Id`"
},
{
"name": "Warning Time",
"expression": "`Warning Time`"
},
{
"name": "Warning Message",
"expression": "`Warning Message`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 1,
"widgetType": "table",
"encodings": {
"columns": [
{
"fieldName": "Pipeline Link",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 100001,
"title": "Pipeline Link",
"allowSearch": true,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 350
},
{
"fieldName": "Pipeline Run Id",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 100002,
"title": "Pipeline Run Id",
"allowSearch": false,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 300
},
{
"fieldName": "Warning Time",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 100005,
"title": "Warning Time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Warning Message",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 100006,
"title": "Warning Message",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
]
},
"invisibleColumns": [
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Name",
"type": "string",
"displayAs": "string",
"order": 100000,
"title": "Pipeline Name",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Run",
"type": "string",
"displayAs": "string",
"order": 100003,
"title": "Pipeline Run",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Run Start Time",
"type": "datetime",
"displayAs": "datetime",
"order": 100004,
"title": "Pipeline Run Start Time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
],
"allowHTMLByDefault": false,
"itemsPerPage": 25,
"paginationSize": "default",
"condensed": true,
"withRowNumber": false,
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Warnings in Pipelines",
"description": "Detected warnings in the pipeline runs"
}
}
},
"position": {
"x": 0,
"y": 18,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "5527b0d7",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "62d03d54",
"fields": [
{
"name": "Pipeline",
"expression": "`Pipeline`"
},
{
"name": "Latest Pipeline Run",
"expression": "`Latest Pipeline Run`"
},
{
"name": "Latest Status",
"expression": "`Latest Status`"
},
{
"name": "Latest error code",
"expression": "`Latest error code`"
},
{
"name": "Latest Create time",
"expression": "`Latest Create time`"
},
{
"name": "Latest End time",
"expression": "`Latest End time`"
},
{
"name": "Latest run errors",
"expression": "`Latest run errors`"
},
{
"name": "Latest run warnings",
"expression": "`Latest run warnings`"
},
{
"name": "Latest error message",
"expression": "`Latest error message`"
},
{
"name": "Latest Total Seconds",
"expression": "`Latest Total Seconds`"
},
{
"name": "Last Updated",
"expression": "`Last Updated`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 1,
"widgetType": "table",
"encodings": {
"columns": [
{
"fieldName": "Pipeline",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 0,
"title": "Pipeline",
"allowSearch": true,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 350
},
{
"fieldName": "Latest Pipeline Run",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 2,
"title": "Latest Pipeline Run",
"allowSearch": false,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 300
},
{
"fieldName": "Latest Status",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 3,
"title": "Latest Status",
"allowSearch": false,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 150
},
{
"fieldName": "Latest error code",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 4,
"title": "Latest error code",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Latest Create time",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 5,
"title": "Latest Create time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Latest End time",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 6,
"title": "Latest End time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Latest run errors",
"numberFormat": "0",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "integer",
"displayAs": "number",
"visible": true,
"order": 9,
"title": "Latest run errors",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Latest run warnings",
"numberFormat": "0",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "integer",
"displayAs": "number",
"visible": true,
"order": 10,
"title": "Latest run warnings",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Latest error message",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 11,
"title": "Latest error message",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Latest Total Seconds",
"numberFormat": "0",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "integer",
"displayAs": "number",
"visible": true,
"order": 12,
"title": "Latest Total Seconds",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Last Updated",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 13,
"title": "Last Updated",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
]
},
"invisibleColumns": [
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Name",
"type": "string",
"displayAs": "string",
"order": 1,
"title": "Pipeline Name",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Status",
"type": "string",
"displayAs": "string",
"order": 7,
"title": "Status",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Latest error log message",
"type": "string",
"displayAs": "string",
"order": 8,
"title": "Latest error log message",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
],
"allowHTMLByDefault": false,
"itemsPerPage": 25,
"paginationSize": "default",
"condensed": true,
"withRowNumber": false,
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Latest Runs by Pipeline ",
"description": "The status of the latest run for each pipeline"
}
}
},
"position": {
"x": 0,
"y": 6,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "pipeline-status-breakdown",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "62d03d54",
"fields": [
{
"name": "count(Status)",
"expression": "COUNT(`Status`)"
},
{
"name": "Status",
"expression": "`Status`"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "pie",
"encodings": {
"angle": {
"fieldName": "count(Status)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Status",
"scale": {
"type": "categorical"
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Pipeline Status Breakdown",
"description": "See \"Latest Runs by Pipeline\" below"
}
}
},
"position": {
"x": 0,
"y": 0,
"width": 2,
"height": 6
}
},
{
"widget": {
"name": "number-of-errors-in-pipelines",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "82c4e3b5",
"fields": [
{
"name": "count(*)",
"expression": "COUNT(`*`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 2,
"widgetType": "counter",
"encodings": {
"value": {
"fieldName": "count(*)",
"style": {
"rules": [
{
"condition": {
"operand": {
"type": "data-value",
"value": "0"
},
"operator": ">"
},
"color": {
"themeColorType": "visualizationColors",
"position": 4
}
}
]
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Number of Errors in Pipelines",
"description": "See \"Errors in Pipelines\" below"
}
}
},
"position": {
"x": 2,
"y": 0,
"width": 2,
"height": 3
}
},
{
"widget": {
"name": "number-of-warnings-in-pipelines",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "c0688ffc",
"fields": [
{
"name": "count(*)",
"expression": "COUNT(`*`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 2,
"widgetType": "counter",
"encodings": {
"value": {
"fieldName": "count(*)",
"style": {
"rules": [
{
"condition": {
"operand": {
"type": "data-value",
"value": "0"
},
"operator": ">"
},
"color": {
"themeColorType": "visualizationColors",
"position": 2
}
}
]
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Number of Warnings in Pipelines",
"description": "See \"Warnings in Pipelines\" below"
}
}
},
"position": {
"x": 2,
"y": 3,
"width": 2,
"height": 3
}
},
{
"widget": {
"name": "mv_w5",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "flow_outcomes",
"fields": [
{
"name": "status",
"expression": "`status`"
},
{
"name": "flow_runs",
"expression": "`flow_runs`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"y": {
"fieldName": "status",
"scale": {
"type": "categorical",
"sort": {
"by": "x-reversed"
}
},
"axis": {
"hideTitle": true
},
"displayName": "status"
},
"x": {
"fieldName": "flow_runs",
"scale": {
"type": "quantitative"
},
"axis": {
"hideTitle": true
},
"displayName": "flow_runs",
"format": {
"type": "number-plain",
"abbreviation": "compact",
"decimalPlaces": {
"type": "max",
"places": 1
}
}
}
},
"frame": {
"showTitle": true,
"title": "Flow outcomes"
}
}
},
"position": {
"x": 0,
"y": 30,
"width": 6,
"height": 6
}
}
],
"pageType": "PAGE_TYPE_CANVAS"
},
{
"name": "0f6e96ec",
"displayName": "Pipelines Metrics",
"layout": [
{
"widget": {
"name": "6e698544",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "2faba057",
"fields": [
{
"name": "Pipeline Name",
"expression": "`Pipeline Name`"
},
{
"name": "hourly(Hour)",
"expression": "DATE_TRUNC(\"HOUR\", `Hour`)"
},
{
"name": "sum(Number of Errors)",
"expression": "SUM(`Number of Errors`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "line",
"encodings": {
"x": {
"fieldName": "hourly(Hour)",
"scale": {
"type": "temporal"
},
"displayName": "Hour"
},
"y": {
"fieldName": "sum(Number of Errors)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Pipeline Name",
"scale": {
"type": "categorical"
},
"displayName": "Pipeline Name"
},
"label": {
"show": false
}
},
"annotations": [],
"frame": {
"showDescription": true,
"showTitle": true,
"title": "Hourly error rate",
"description": "The number of errors per hour across selected pipelines"
}
}
},
"position": {
"x": 0,
"y": 2,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "c972afff",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "82c4e3b5",
"fields": [
{
"name": "count(*)",
"expression": "COUNT(`*`)"
},
{
"name": "Error Code",
"expression": "`Error Code`"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "pie",
"encodings": {
"angle": {
"fieldName": "count(*)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Error Code",
"scale": {
"type": "categorical"
}
},
"label": {
"show": true
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Distribution of errors codes",
"description": "Relative share of different types of errors across pipeline runs"
}
}
},
"position": {
"x": 0,
"y": 8,
"width": 6,
"height": 7
}
},
{
"widget": {
"name": "dd54ef0b",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "19d796af",
"fields": [
{
"name": "Status String",
"expression": "`Status String`"
},
{
"name": "Pipeline Name",
"expression": "`Pipeline Name`"
},
{
"name": "minutely(Create time)",
"expression": "DATE_TRUNC(\"MINUTE\", `Create time`)"
},
{
"name": "sum(Total seconds)",
"expression": "SUM(`Total seconds`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "minutely(Create time)",
"scale": {
"type": "categorical"
}
},
"y": {
"fieldName": "sum(Total seconds)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Status String",
"scale": {
"type": "categorical",
"mappings": [
{
"value": "COMPLETED",
"color": {
"themeColorType": "visualizationColors",
"position": 3
}
},
{
"value": "FAILED",
"color": {
"themeColorType": "visualizationColors",
"position": 4
}
},
{
"value": "WAITING_FOR_RESOURCES",
"color": {
"themeColorType": "visualizationColors",
"position": 5
}
}
]
}
},
"extra": [
{
"fieldName": "Pipeline Name"
}
]
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Total time of pipeline runs",
"description": "Total execution time and final status for recent pipeline runs"
}
}
},
"position": {
"x": 0,
"y": 25,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "c1f6e5a8",
"multilineTextboxSpec": {
"lines": ["# Errors"]
}
},
"position": {
"x": 0,
"y": 0,
"width": 6,
"height": 2
}
},
{
"widget": {
"name": "f7a042ed",
"multilineTextboxSpec": {
"lines": ["# Pipeline Durations"]
}
},
"position": {
"x": 0,
"y": 23,
"width": 6,
"height": 2
}
},
{
"widget": {
"name": "6b06b493",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "19d796af",
"fields": [
{
"name": "Pipeline Name",
"expression": "`Pipeline Name`"
},
{
"name": "minutely(Create time)",
"expression": "DATE_TRUNC(\"MINUTE\", `Create time`)"
},
{
"name": "sum(Starting seconds)",
"expression": "SUM(`Starting seconds`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "minutely(Create time)",
"scale": {
"type": "categorical"
}
},
"y": {
"fieldName": "sum(Starting seconds)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Pipeline Name",
"scale": {
"type": "categorical"
}
},
"extra": [
{
"fieldName": "Pipeline Name"
}
]
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Starting time of pipeline runs",
"description": "Captures the time waiting for resources to start the pipeline"
}
}
},
"position": {
"x": 0,
"y": 31,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "77547bb3",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "19d796af",
"fields": [
{
"name": "Pipeline Name",
"expression": "`Pipeline Name`"
},
{
"name": "minutely(Create time)",
"expression": "DATE_TRUNC(\"MINUTE\", `Create time`)"
},
{
"name": "sum(Initialization seconds)",
"expression": "SUM(`Initialization seconds`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "minutely(Create time)",
"scale": {
"type": "categorical"
}
},
"y": {
"fieldName": "sum(Initialization seconds)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Pipeline Name",
"scale": {
"type": "categorical"
}
},
"extra": [
{
"fieldName": "Pipeline Name"
}
]
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Initialization time of pipeline runs",
"description": "Captures the time intializing the pipeline"
}
}
},
"position": {
"x": 0,
"y": 37,
"width": 6,
"height": 7
}
},
{
"widget": {
"name": "530d0177",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "19d796af",
"fields": [
{
"name": "Pipeline Name",
"expression": "`Pipeline Name`"
},
{
"name": "minutely(Create time)",
"expression": "DATE_TRUNC(\"MINUTE\", `Create time`)"
},
{
"name": "sum(Running seconds)",
"expression": "SUM(`Running seconds`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "minutely(Create time)",
"scale": {
"type": "categorical"
}
},
"y": {
"fieldName": "sum(Running seconds)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Pipeline Name",
"scale": {
"type": "categorical"
}
},
"extra": [
{
"fieldName": "Pipeline Name"
}
]
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Execution time of pipeline runs",
"description": "Captures the time evaluating the data flow graph of the pipeline"
}
}
},
"position": {
"x": 0,
"y": 44,
"width": 6,
"height": 7
}
},
{
"widget": {
"name": "945b778a",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "2660d018",
"fields": [
{
"name": "Pipeline Name",
"expression": "`Pipeline Name`"
},
{
"name": "hourly(Event Time)",
"expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
},
{
"name": "sum(Total Changes)",
"expression": "SUM(`Total Changes`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "line",
"encodings": {
"x": {
"fieldName": "hourly(Event Time)",
"scale": {
"type": "temporal"
}
},
"y": {
"fieldName": "sum(Total Changes)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Pipeline Name",
"scale": {
"type": "categorical"
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Hourly rows changed (by pipeline)",
"description": "Rows appended, upserted, and deleted per hour, by pipeline"
}
}
},
"position": {
"x": 0,
"y": 17,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "bc0b65a1",
"multilineTextboxSpec": {
"lines": ["# Throughput and Rates"]
}
},
"position": {
"x": 0,
"y": 15,
"width": 6,
"height": 2
}
},
{
"widget": {
"name": "mv_w6",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "hygiene",
"fields": [
{
"name": "item",
"expression": "`item`"
},
{
"name": "events",
"expression": "`events`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"y": {
"fieldName": "item",
"scale": {
"type": "categorical",
"sort": {
"by": "x-reversed"
}
},
"axis": {
"hideTitle": true
},
"displayName": "item"
},
"x": {
"fieldName": "events",
"scale": {
"type": "quantitative"
},
"axis": {
"hideTitle": true
},
"displayName": "events",
"format": {
"type": "number-plain",
"abbreviation": "compact",
"decimalPlaces": {
"type": "max",
"places": 1
}
}
}
},
"frame": {
"showTitle": true,
"title": "Deprecated / unsupported / advisory (by feature)"
}
}
},
"position": {
"x": 0,
"y": 51,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "mv_w15",
"queries": [
{
"name": "src",
"query": {
"datasetName": "util_groupby",
"fields": [
{
"name": "opt",
"expression": "`opt`"
}
],
"disaggregated": false
}
},
{
"name": "p_dd_util",
"query": {
"datasetName": "dd_util",
"parameters": [
{
"name": "param_util_groupby",
"keyword": "param_util_groupby"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 2,
"widgetType": "filter-single-select",
"encodings": {
"fields": [
{
"fieldName": "opt",
"displayName": "opt",
"queryName": "src"
},
{
"parameterName": "param_util_groupby",
"queryName": "p_dd_util"
}
]
},
"frame": {
"showTitle": true,
"title": "Group by cluster"
},
"selection": {
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "all clusters"
}
]
}
}
}
}
},
"position": {
"x": 0,
"y": 57,
"width": 2,
"height": 1
}
},
{
"widget": {
"name": "mv_w16",
"queries": [
{
"name": "src",
"query": {
"datasetName": "util_grain",
"fields": [
{
"name": "grain",
"expression": "`grain`"
}
],
"disaggregated": false
}
},
{
"name": "p_dd_util",
"query": {
"datasetName": "dd_util",
"parameters": [
{
"name": "param_util_grain",
"keyword": "param_util_grain"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 2,
"widgetType": "filter-single-select",
"encodings": {
"fields": [
{
"fieldName": "grain",
"displayName": "grain",
"queryName": "src"
},
{
"parameterName": "param_util_grain",
"queryName": "p_dd_util"
}
]
},
"frame": {
"showTitle": true,
"title": "Time granularity"
},
"selection": {
"defaultSelection": {
"values": {
"dataType": "STRING",
"values": [
{
"value": "hour"
}
]
}
}
}
}
},
"position": {
"x": 2,
"y": 57,
"width": 2,
"height": 1
}
},
{
"widget": {
"name": "mv_w17",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "dd_util",
"fields": [
{
"name": "ts",
"expression": "`ts`"
},
{
"name": "slot_utilization_pct",
"expression": "`slot_utilization_pct`"
},
{
"name": "cluster",
"expression": "`cluster`"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "line",
"encodings": {
"x": {
"fieldName": "ts",
"scale": {
"type": "temporal"
},
"format": {
"type": "date-time",
"date": "locale-short-month",
"time": "none"
},
"axis": {
"hideTitle": true
},
"displayName": "ts"
},
"y": {
"fieldName": "slot_utilization_pct",
"scale": {
"type": "quantitative"
},
"axis": {
"hideTitle": true
},
"displayName": "slot_utilization_pct",
"format": {
"type": "number-plain",
"abbreviation": "compact",
"decimalPlaces": {
"type": "max",
"places": 1
}
}
},
"color": {
"fieldName": "cluster",
"scale": {
"type": "categorical"
},
"legend": {
"position": "bottom",
"hideTitle": true
},
"displayName": "cluster"
}
},
"frame": {
"showTitle": true,
"title": "Cluster utilization %"
}
}
},
"position": {
"x": 0,
"y": 58,
"width": 6,
"height": 6
}
}
],
"pageType": "PAGE_TYPE_CANVAS"
},
{
"name": "6701599f",
"displayName": "Tables Health",
"layout": [
{
"widget": {
"name": "057ead9b",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "248b74c6",
"fields": [
{
"name": "Pipeline Link",
"expression": "`Pipeline Link`"
},
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "Latest Run",
"expression": "`Latest Run`"
},
{
"name": "Latest State",
"expression": "`Latest State`"
},
{
"name": "Latest Changes Time",
"expression": "`Latest Changes Time`"
},
{
"name": "Seconds Since Latest Change",
"expression": "`Seconds Since Latest Change`"
},
{
"name": "Latest Schema",
"expression": "`Latest Schema`"
},
{
"name": "Time of Latest Error",
"expression": "`Time of Latest Error`"
},
{
"name": "Code of Latest Error",
"expression": "`Code of Latest Error`"
},
{
"name": "Message of Latest Error",
"expression": "`Message of Latest Error`"
},
{
"name": "Log Message of Latest Error",
"expression": "`Log Message of Latest Error`"
},
{
"name": "Latest Pipeline Run with Error",
"expression": "`Latest Pipeline Run with Error`"
},
{
"name": "Last Updated",
"expression": "`Last Updated`"
},
{
"name": "Seconds Since Latest Status Update",
"expression": "`Seconds Since Latest Status Update`"
},
{
"name": "Last Data Refresh Time",
"expression": "`Last Data Refresh Time`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 1,
"widgetType": "table",
"encodings": {
"columns": [
{
"fieldName": "Pipeline Link",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 1,
"title": "Pipeline Link",
"allowSearch": true,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 350
},
{
"fieldName": "Target Table",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 2,
"title": "Target Table",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Latest Run",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 3,
"title": "Latest Run",
"allowSearch": false,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 300
},
{
"fieldName": "Latest State",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 4,
"title": "Latest State",
"allowSearch": true,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 125
},
{
"fieldName": "Latest Changes Time",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 5,
"title": "Latest Changes Time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Seconds Since Latest Change",
"numberFormat": "0",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "integer",
"displayAs": "number",
"visible": true,
"order": 6,
"title": "Seconds Since Latest Change",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Latest Schema",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "complex",
"displayAs": "json",
"visible": true,
"order": 7,
"title": "Latest Schema",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Time of Latest Error",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 8,
"title": "Time of Latest Error",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Code of Latest Error",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 9,
"title": "Code of Latest Error",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Message of Latest Error",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 10,
"title": "Message of Latest Error",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Log Message of Latest Error",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 11,
"title": "Log Message of Latest Error",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Latest Pipeline Run with Error",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 12,
"title": "Latest Pipeline Run with Error",
"allowSearch": false,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 325
},
{
"fieldName": "Last Updated",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 13,
"title": "Last Updated",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Seconds Since Latest Status Update",
"numberFormat": "0",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "integer",
"displayAs": "number",
"visible": true,
"order": 14,
"title": "Seconds Since Latest Status Update",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Last Data Refresh Time",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 15,
"title": "Last Data Refresh Time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
]
},
"invisibleColumns": [
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Name",
"type": "string",
"displayAs": "string",
"order": 0,
"title": "Pipeline Name",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
],
"allowHTMLByDefault": false,
"itemsPerPage": 25,
"paginationSize": "default",
"condensed": true,
"withRowNumber": false,
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Table Status",
"description": "Latest Status of Monitored Tables"
}
}
},
"position": {
"x": 0,
"y": 6,
"width": 6,
"height": 10
}
},
{
"widget": {
"name": "cc2c8376",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "7103e200",
"fields": [
{
"name": "Pipeline Link",
"expression": "`Pipeline Link`"
},
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "Pipeline Run",
"expression": "`Pipeline Run`"
},
{
"name": "Table State",
"expression": "`Table State`"
},
{
"name": "Error Code",
"expression": "`Error Code`"
},
{
"name": "Error Message",
"expression": "`Error Message`"
},
{
"name": "Error Message in Log",
"expression": "`Error Message in Log`"
},
{
"name": "Error Time",
"expression": "`Error Time`"
},
{
"name": "Latest Schema",
"expression": "`Latest Schema`"
},
{
"name": "Pipeline Run Create Time",
"expression": "`Pipeline Run Create Time`"
},
{
"name": "Pipeline Run End Time",
"expression": "`Pipeline Run End Time`"
},
{
"name": "Pipeline Run State",
"expression": "`Pipeline Run State`"
},
{
"name": "Last Updated",
"expression": "`Last Updated`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 1,
"widgetType": "table",
"encodings": {
"columns": [
{
"fieldName": "Pipeline Link",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 1,
"title": "Pipeline Link",
"allowSearch": true,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 350
},
{
"fieldName": "Target Table",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 2,
"title": "Target Table",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Pipeline Run",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 3,
"title": "Pipeline Run",
"allowSearch": false,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"description": "",
"defaultColumnWidth": 325
},
{
"fieldName": "Table State",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 5,
"title": "Table State",
"allowSearch": false,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 125
},
{
"fieldName": "Error Code",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 6,
"title": "Error Code",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Error Message",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 7,
"title": "Error Message",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Error Message in Log",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 8,
"title": "Error Message in Log",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Error Time",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 9,
"title": "Error Time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Latest Schema",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "complex",
"displayAs": "json",
"visible": true,
"order": 10,
"title": "Latest Schema",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Pipeline Run Create Time",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 11,
"title": "Pipeline Run Create Time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Pipeline Run End Time",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 12,
"title": "Pipeline Run End Time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Pipeline Run State",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 13,
"title": "Pipeline Run State",
"allowSearch": false,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 150
},
{
"fieldName": "Last Updated",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 14,
"title": "Last Updated",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
]
},
"invisibleColumns": [
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Name",
"type": "string",
"displayAs": "string",
"order": 0,
"title": "Pipeline Name",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
],
"allowHTMLByDefault": false,
"itemsPerPage": 25,
"paginationSize": "default",
"condensed": true,
"withRowNumber": false,
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Tables Status per Pipeline Run",
"description": "Table statuses for recent pipeline runs"
}
}
},
"position": {
"x": 0,
"y": 46,
"width": 6,
"height": 9
}
},
{
"widget": {
"name": "0ec2e19e",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "3b8b7d86",
"fields": [
{
"name": "Pipeline Link",
"expression": "`Pipeline Link`"
},
{
"name": "Pipeline Run",
"expression": "`Pipeline Run`"
},
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "Error Code",
"expression": "`Error Code`"
},
{
"name": "Error Message",
"expression": "`Error Message`"
},
{
"name": "Error Log Message",
"expression": "`Error Log Message`"
},
{
"name": "Error Details",
"expression": "`Error Details`"
},
{
"name": "Error Time",
"expression": "`Error Time`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 1,
"widgetType": "table",
"encodings": {
"columns": [
{
"fieldName": "Pipeline Link",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 1,
"title": "Pipeline Link",
"allowSearch": true,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 350
},
{
"fieldName": "Pipeline Run",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 3,
"title": "Pipeline Run",
"allowSearch": false,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 325
},
{
"fieldName": "Target Table",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 4,
"title": "Target Table",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Error Code",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 5,
"title": "Error Code",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Error Message",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 6,
"title": "Error Message",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Error Log Message",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 7,
"title": "Error Log Message",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Error Details",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "complex",
"displayAs": "json",
"visible": true,
"order": 8,
"title": "Error Details",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Error Time",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 9,
"title": "Error Time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
]
},
"invisibleColumns": [
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Name",
"type": "string",
"displayAs": "string",
"order": 0,
"title": "Pipeline Name",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Run Id",
"type": "string",
"displayAs": "string",
"order": 2,
"title": "Pipeline Run Id",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Start Time",
"type": "datetime",
"displayAs": "datetime",
"order": 10,
"title": "Pipeline Start Time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
],
"allowHTMLByDefault": false,
"itemsPerPage": 25,
"paginationSize": "default",
"condensed": true,
"withRowNumber": false,
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Table Errors",
"description": "Table error details in recent pipeline runs"
}
}
},
"position": {
"x": 0,
"y": 16,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "be0d5fae",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "8897fc81",
"fields": [
{
"name": "Pipeline Link",
"expression": "`Pipeline Link`"
},
{
"name": "Pipeline Run",
"expression": "`Pipeline Run`"
},
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "Warning Log Message",
"expression": "`Warning Log Message`"
},
{
"name": "Warning Time",
"expression": "`Warning Time`"
}
],
"disaggregated": true
}
}
],
"spec": {
"version": 1,
"widgetType": "table",
"encodings": {
"columns": [
{
"fieldName": "Pipeline Link",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 1,
"title": "Pipeline Link",
"allowSearch": true,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 350
},
{
"fieldName": "Pipeline Run",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 3,
"title": "Pipeline Run",
"allowSearch": false,
"alignContent": "left",
"allowHTML": true,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false,
"defaultColumnWidth": 325
},
{
"fieldName": "Target Table",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 4,
"title": "Target Table",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Warning Log Message",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "string",
"displayAs": "string",
"visible": true,
"order": 5,
"title": "Warning Log Message",
"allowSearch": true,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"fieldName": "Warning Time",
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"type": "datetime",
"displayAs": "datetime",
"visible": true,
"order": 6,
"title": "Warning Time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
]
},
"invisibleColumns": [
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Name",
"type": "string",
"displayAs": "string",
"order": 0,
"title": "Pipeline Name",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Run Id",
"type": "string",
"displayAs": "string",
"order": 2,
"title": "Pipeline Run Id",
"allowSearch": false,
"alignContent": "left",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
},
{
"dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
"booleanValues": ["false", "true"],
"imageUrlTemplate": "{{ @ }}",
"imageTitleTemplate": "{{ @ }}",
"imageWidth": "",
"imageHeight": "",
"linkUrlTemplate": "{{ @ }}",
"linkTextTemplate": "{{ @ }}",
"linkTitleTemplate": "{{ @ }}",
"linkOpenInNewTab": true,
"name": "Pipeline Start Time",
"type": "datetime",
"displayAs": "datetime",
"order": 7,
"title": "Pipeline Start Time",
"allowSearch": false,
"alignContent": "right",
"allowHTML": false,
"highlightLinks": false,
"useMonospaceFont": false,
"preserveWhitespace": false
}
],
"allowHTMLByDefault": false,
"itemsPerPage": 25,
"paginationSize": "default",
"condensed": true,
"withRowNumber": false,
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Table Warnings",
"description": "Table warnings details in recent pipeline runs"
}
}
},
"position": {
"x": 0,
"y": 40,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "expectation-failures",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "f786fd7e",
"fields": [
{
"name": "secondly(Check Time)",
"expression": "DATE_TRUNC(\"SECOND\", `Check Time`)"
},
{
"name": "sum(Rows Failed)",
"expression": "SUM(`Rows Failed`)"
},
{
"name": "max(Failure Percentage)",
"expression": "MAX(`Failure Percentage`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 1,
"widgetType": "combo",
"encodings": {
"x": {
"fieldName": "secondly(Check Time)",
"scale": {
"type": "categorical"
}
},
"label": {
"show": false
},
"y": {
"primary": {
"fields": [
{
"fieldName": "sum(Rows Failed)"
}
],
"scale": {
"type": "quantitative"
}
},
"secondary": {
"fields": [
{
"fieldName": "max(Failure Percentage)"
}
],
"scale": {
"type": "quantitative"
}
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Expectation Failures",
"description": "Expectation failures across all selected tables and expectations"
}
}
},
"position": {
"x": 0,
"y": 22,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "break-down-by-expectation",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "f786fd7e",
"fields": [
{
"name": "sum(Rows Failed)",
"expression": "SUM(`Rows Failed`)"
},
{
"name": "Expectation Name",
"expression": "`Expectation Name`"
},
{
"name": "sum(Rows Passed)",
"expression": "SUM(`Rows Passed`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "pie",
"encodings": {
"angle": {
"fieldName": "sum(Rows Failed)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Expectation Name",
"scale": {
"type": "categorical"
}
},
"extra": [
{
"fieldName": "sum(Rows Passed)"
}
]
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Expectation failure break down by expectation",
"description": "The share of failed rows by expectation name"
}
}
},
"position": {
"x": 0,
"y": 28,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "803d2c81",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "f786fd7e",
"fields": [
{
"name": "sum(Rows Failed)",
"expression": "SUM(`Rows Failed`)"
},
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "sum(Rows Passed)",
"expression": "SUM(`Rows Passed`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "pie",
"encodings": {
"angle": {
"fieldName": "sum(Rows Failed)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Target Table",
"scale": {
"type": "categorical"
}
},
"extra": [
{
"fieldName": "sum(Rows Passed)"
}
]
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Expectation failure break down by table",
"description": "The share of failed rows by table"
}
}
},
"position": {
"x": 0,
"y": 34,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "table-statuses-breakdown",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "248b74c6",
"fields": [
{
"name": "count(State)",
"expression": "COUNT(`State`)"
},
{
"name": "State",
"expression": "`State`"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "pie",
"encodings": {
"angle": {
"fieldName": "count(State)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "State",
"scale": {
"type": "categorical"
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Table Statuses Breakdown",
"description": "See \"Table Status\" below"
}
}
},
"position": {
"x": 0,
"y": 0,
"width": 2,
"height": 6
}
},
{
"widget": {
"name": "table-errors",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "3b8b7d86",
"fields": [
{
"name": "count(*)",
"expression": "COUNT(`*`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 2,
"widgetType": "counter",
"encodings": {
"value": {
"fieldName": "count(*)",
"style": {
"rules": [
{
"condition": {
"operand": {
"type": "data-value",
"value": "0"
},
"operator": ">"
},
"color": {
"themeColorType": "visualizationColors",
"position": 4
}
}
]
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Number of Table Errors",
"description": "See \"Table Errors\" below"
}
}
},
"position": {
"x": 2,
"y": 0,
"width": 1,
"height": 3
}
},
{
"widget": {
"name": "number-table-warnings",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "8897fc81",
"fields": [
{
"name": "count(*)",
"expression": "COUNT(`*`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 2,
"widgetType": "counter",
"encodings": {
"value": {
"fieldName": "count(*)",
"style": {
"rules": [
{
"condition": {
"operand": {
"type": "data-value",
"value": "0"
},
"operator": ">"
},
"color": {
"themeColorType": "visualizationColors",
"position": 2
}
}
]
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Number of Table Warnings",
"description": "See \"Table Warnings\" below"
}
}
},
"position": {
"x": 2,
"y": 3,
"width": 1,
"height": 3
}
}
],
"pageType": "PAGE_TYPE_CANVAS"
},
{
"name": "f0c5d746",
"displayName": "Global Filters",
"layout": [
{
"widget": {
"name": "5315d004",
"queries": [
{
"name": "dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b32acc049781d0cf011_pipeline_id",
"query": {
"datasetName": "6740610b",
"fields": [
{
"name": "pipeline_id",
"expression": "`pipeline_id`"
},
{
"name": "pipeline_id_associativity",
"expression": "COUNT_IF(`associative_filter_predicate_group`)"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b74b28865962df14f44_pipeline_id",
"query": {
"datasetName": "0ac0bb93",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b9aa9c13d1f77991e37_pipeline_id",
"query": {
"datasetName": "8db826d7",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461ba4b22cc5d22af91321_pipeline_id",
"query": {
"datasetName": "b18ee129",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b54b96f2b00fb18b522_pipeline_id",
"query": {
"datasetName": "5e3eb18f",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bb2a2c9b3b4e6334bc3_pipeline_id",
"query": {
"datasetName": "19d796af",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b0aad0f649404f8cb25_pipeline_id",
"query": {
"datasetName": "c91619f8",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461a9bbe5fbf4995bdc44f_pipeline_id",
"query": {
"datasetName": "82c4e3b5",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b8d9e292d0d2dc67dd6_pipeline_id",
"query": {
"datasetName": "c0688ffc",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bbdb4e1f9e9952a2684_pipeline_id",
"query": {
"datasetName": "3b8b7d86",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b3cb83f2bd346127021_pipeline_id",
"query": {
"datasetName": "8897fc81",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b488f6ffac402178c47_pipeline_id",
"query": {
"datasetName": "2faba057",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bd6911ce40b60eaf9a5_pipeline_id",
"query": {
"datasetName": "62d03d54",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b1ab6fe6f2600b36759_pipeline_id",
"query": {
"datasetName": "2660d018",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b67bbebc173149c5e91_pipeline_id",
"query": {
"datasetName": "248b74c6",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b7facb56e496d7acee2_pipeline_id",
"query": {
"datasetName": "7103e200",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461be59b3be5a31cd1c7c2_pipeline_id",
"query": {
"datasetName": "b72dbc41",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bf3ba5f4569c02f5f10_pipeline_id",
"query": {
"datasetName": "e9270b22",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bff8c74816d8fb2d26d_pipeline_id",
"query": {
"datasetName": "f786fd7e",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461c0ca20f659ce33312ee_pipeline_id",
"query": {
"datasetName": "bc1ad993",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6c03f63139e8465a3bb9d4b4785_pipeline_id",
"query": {
"datasetName": "a189e440",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_moved_flow_outcomes_pipeline_id",
"query": {
"datasetName": "flow_outcomes",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_moved_hygiene_pipeline_id",
"query": {
"datasetName": "hygiene",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
},
{
"name": "parameter_moved_dd_util_pipeline_id",
"query": {
"datasetName": "dd_util",
"parameters": [
{
"name": "pipeline_id",
"keyword": "pipeline_id"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 2,
"widgetType": "filter-single-select",
"encodings": {
"fields": [
{
"fieldName": "pipeline_id",
"queryName": "dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b32acc049781d0cf011_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b74b28865962df14f44_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b9aa9c13d1f77991e37_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461ba4b22cc5d22af91321_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b54b96f2b00fb18b522_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bb2a2c9b3b4e6334bc3_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b0aad0f649404f8cb25_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461a9bbe5fbf4995bdc44f_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b8d9e292d0d2dc67dd6_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bbdb4e1f9e9952a2684_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b3cb83f2bd346127021_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b488f6ffac402178c47_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bd6911ce40b60eaf9a5_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b1ab6fe6f2600b36759_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b67bbebc173149c5e91_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b7facb56e496d7acee2_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461be59b3be5a31cd1c7c2_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bf3ba5f4569c02f5f10_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bff8c74816d8fb2d26d_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461c0ca20f659ce33312ee_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6c03f63139e8465a3bb9d4b4785_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_moved_flow_outcomes_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_moved_hygiene_pipeline_id"
},
{
"parameterName": "pipeline_id",
"queryName": "parameter_moved_dd_util_pipeline_id"
}
]
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Pipelines",
"description": "Select a pipeline to observe with selected tags"
}
}
},
"position": {
"x": 0,
"y": 2,
"width": 1,
"height": 2
}
},
{
"widget": {
"name": "38094f88",
"queries": [
{
"name": "dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1087b64f359a100390be_table_name",
"query": {
"datasetName": "8db826d7",
"fields": [
{
"name": "table_name",
"expression": "`table_name`"
},
{
"name": "table_name_associativity",
"expression": "COUNT_IF(`associative_filter_predicate_group`)"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1091ba6a3afac7047a0f_table_name",
"query": {
"datasetName": "b18ee129",
"parameters": [
{
"name": "table_name",
"keyword": "table_name"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10ab970ed31098c78e69_table_name",
"query": {
"datasetName": "3b8b7d86",
"parameters": [
{
"name": "table_name",
"keyword": "table_name"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1024aea58a5922819fe1_table_name",
"query": {
"datasetName": "8897fc81",
"parameters": [
{
"name": "table_name",
"keyword": "table_name"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b100c8f076556b0de8fb5_table_name",
"query": {
"datasetName": "2660d018",
"parameters": [
{
"name": "table_name",
"keyword": "table_name"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b105597260198f0433892_table_name",
"query": {
"datasetName": "248b74c6",
"parameters": [
{
"name": "table_name",
"keyword": "table_name"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b106e85f1f1d9342869c3_table_name",
"query": {
"datasetName": "7103e200",
"parameters": [
{
"name": "table_name",
"keyword": "table_name"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10c4a4c246ef94cd279b_table_name",
"query": {
"datasetName": "b72dbc41",
"parameters": [
{
"name": "table_name",
"keyword": "table_name"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e926b1b3daa57707d853060ae_table_name",
"query": {
"datasetName": "e9270b22",
"parameters": [
{
"name": "table_name",
"keyword": "table_name"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b6904a0e12d39620b3d5d0055c4c_table_name",
"query": {
"datasetName": "f786fd7e",
"parameters": [
{
"name": "table_name",
"keyword": "table_name"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b690d5ab1755bf99123705eecdfe_table_name",
"query": {
"datasetName": "bc1ad993",
"parameters": [
{
"name": "table_name",
"keyword": "table_name"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 2,
"widgetType": "filter-single-select",
"encodings": {
"fields": [
{
"fieldName": "table_name",
"queryName": "dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1087b64f359a100390be_table_name"
},
{
"parameterName": "table_name",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1091ba6a3afac7047a0f_table_name"
},
{
"parameterName": "table_name",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10ab970ed31098c78e69_table_name"
},
{
"parameterName": "table_name",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1024aea58a5922819fe1_table_name"
},
{
"parameterName": "table_name",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b100c8f076556b0de8fb5_table_name"
},
{
"parameterName": "table_name",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b105597260198f0433892_table_name"
},
{
"parameterName": "table_name",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b106e85f1f1d9342869c3_table_name"
},
{
"parameterName": "table_name",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10c4a4c246ef94cd279b_table_name"
},
{
"parameterName": "table_name",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e926b1b3daa57707d853060ae_table_name"
},
{
"parameterName": "table_name",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b6904a0e12d39620b3d5d0055c4c_table_name"
},
{
"parameterName": "table_name",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b690d5ab1755bf99123705eecdfe_table_name"
}
]
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Tables",
"description": "Select a table to observe"
}
}
},
"position": {
"x": 0,
"y": 4,
"width": 1,
"height": 2
}
},
{
"widget": {
"name": "cc47542d",
"queries": [
{
"name": "dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1049bab36f0d89f9f773_event_timestamp",
"query": {
"datasetName": "5e3eb18f",
"fields": [
{
"name": "event_timestamp",
"expression": "`event_timestamp`"
},
{
"name": "event_timestamp_associativity",
"expression": "COUNT_IF(`associative_filter_predicate_group`)"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b109daa983b86d2ea0e85_event_range",
"query": {
"datasetName": "19d796af",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549a1f9682759c5da45cd926_event_range",
"query": {
"datasetName": "82c4e3b5",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b107b8ce250ca5a5ac326_event_range",
"query": {
"datasetName": "c0688ffc",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10ab970ed31098c78e69_event_range",
"query": {
"datasetName": "3b8b7d86",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1024aea58a5922819fe1_event_range",
"query": {
"datasetName": "8897fc81",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b103181194647c214b1aa_event_range",
"query": {
"datasetName": "2faba057",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10b9badc5ee6b91b8d89_event_range",
"query": {
"datasetName": "62d03d54",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b100c8f076556b0de8fb5_event_range",
"query": {
"datasetName": "2660d018",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b105597260198f0433892_event_range",
"query": {
"datasetName": "248b74c6",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b106e85f1f1d9342869c3_event_range",
"query": {
"datasetName": "7103e200",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10c4a4c246ef94cd279b_event_range",
"query": {
"datasetName": "b72dbc41",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e926b1b3daa57707d853060ae_event_range",
"query": {
"datasetName": "e9270b22",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b6904a0e12d39620b3d5d0055c4c_event_range",
"query": {
"datasetName": "f786fd7e",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b690d5ab1755bf99123705eecdfe_event_range",
"query": {
"datasetName": "bc1ad993",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_moved_flow_outcomes_event_range",
"query": {
"datasetName": "flow_outcomes",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_moved_hygiene_event_range",
"query": {
"datasetName": "hygiene",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
},
{
"name": "parameter_moved_dd_util_event_range",
"query": {
"datasetName": "dd_util",
"parameters": [
{
"name": "event_range",
"keyword": "event_range"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 2,
"widgetType": "filter-date-range-picker",
"encodings": {
"fields": [
{
"fieldName": "event_timestamp",
"queryName": "dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1049bab36f0d89f9f773_event_timestamp"
},
{
"parameterName": "event_range",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b109daa983b86d2ea0e85_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549a1f9682759c5da45cd926_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b107b8ce250ca5a5ac326_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10ab970ed31098c78e69_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1024aea58a5922819fe1_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b103181194647c214b1aa_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10b9badc5ee6b91b8d89_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b100c8f076556b0de8fb5_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b105597260198f0433892_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b106e85f1f1d9342869c3_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10c4a4c246ef94cd279b_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e926b1b3daa57707d853060ae_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b6904a0e12d39620b3d5d0055c4c_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b690d5ab1755bf99123705eecdfe_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_moved_flow_outcomes_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_moved_hygiene_event_range"
},
{
"parameterName": "event_range",
"queryName": "parameter_moved_dd_util_event_range"
}
]
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Date/Time Range",
"description": "Select a event log data/time range to observe"
},
"selection": {
"defaultSelection": {
"range": {
"dataType": "DATETIME",
"min": {
"value": "now-24h/h"
},
"max": {
"value": "now/h"
}
}
}
}
}
},
"position": {
"x": 0,
"y": 6,
"width": 1,
"height": 2
}
},
{
"widget": {
"name": "pipeline-tags",
"queries": [
{
"name": "dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b299faa569d93cdfe85_tag_value",
"query": {
"datasetName": "5ef35e45",
"fields": [
{
"name": "tag_value",
"expression": "`tag_value`"
},
{
"name": "tag_value_associativity",
"expression": "COUNT_IF(`associative_filter_predicate_group`)"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b32acc049781d0cf011_tag_value",
"query": {
"datasetName": "6740610b",
"parameters": [
{
"name": "tag_value",
"keyword": "tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b74b28865962df14f44_pipeline_tag_value",
"query": {
"datasetName": "0ac0bb93",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b9aa9c13d1f77991e37_pipeline_tag_value",
"query": {
"datasetName": "8db826d7",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461ba4b22cc5d22af91321_pipeline_tag_value",
"query": {
"datasetName": "b18ee129",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461a9bbe5fbf4995bdc44f_pipeline_tag_value",
"query": {
"datasetName": "82c4e3b5",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b8d9e292d0d2dc67dd6_pipeline_tag_value",
"query": {
"datasetName": "c0688ffc",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bb2a2c9b3b4e6334bc3_pipeline_tag_value",
"query": {
"datasetName": "19d796af",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bd6911ce40b60eaf9a5_pipeline_tag_value",
"query": {
"datasetName": "62d03d54",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b0aad0f649404f8cb25_pipeline_tag_value",
"query": {
"datasetName": "c91619f8",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b488f6ffac402178c47_pipeline_tag_value",
"query": {
"datasetName": "2faba057",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b1ab6fe6f2600b36759_pipeline_tag_value",
"query": {
"datasetName": "2660d018",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b3cb83f2bd346127021_pipeline_tag_value",
"query": {
"datasetName": "8897fc81",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b67bbebc173149c5e91_pipeline_tag_value",
"query": {
"datasetName": "248b74c6",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b7facb56e496d7acee2_pipeline_tag_value",
"query": {
"datasetName": "7103e200",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bbdb4e1f9e9952a2684_pipeline_tag_value",
"query": {
"datasetName": "3b8b7d86",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461be59b3be5a31cd1c7c2_pipeline_tag_value",
"query": {
"datasetName": "b72dbc41",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bf3ba5f4569c02f5f10_pipeline_tag_value",
"query": {
"datasetName": "e9270b22",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bff8c74816d8fb2d26d_pipeline_tag_value",
"query": {
"datasetName": "f786fd7e",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461c0ca20f659ce33312ee_pipeline_tag_value",
"query": {
"datasetName": "bc1ad993",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6c03f63139e8465a3bb9d4b4785_pipeline_tag_value",
"query": {
"datasetName": "a189e440",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_moved_flow_outcomes_pipeline_tag_value",
"query": {
"datasetName": "flow_outcomes",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_moved_hygiene_pipeline_tag_value",
"query": {
"datasetName": "hygiene",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
},
{
"name": "parameter_moved_dd_util_pipeline_tag_value",
"query": {
"datasetName": "dd_util",
"parameters": [
{
"name": "pipeline_tag_value",
"keyword": "pipeline_tag_value"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 2,
"widgetType": "filter-single-select",
"encodings": {
"fields": [
{
"fieldName": "tag_value",
"queryName": "dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b299faa569d93cdfe85_tag_value"
},
{
"parameterName": "tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b32acc049781d0cf011_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b74b28865962df14f44_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b9aa9c13d1f77991e37_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461ba4b22cc5d22af91321_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461a9bbe5fbf4995bdc44f_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b8d9e292d0d2dc67dd6_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bb2a2c9b3b4e6334bc3_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bd6911ce40b60eaf9a5_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b0aad0f649404f8cb25_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b488f6ffac402178c47_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b1ab6fe6f2600b36759_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b3cb83f2bd346127021_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b67bbebc173149c5e91_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b7facb56e496d7acee2_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bbdb4e1f9e9952a2684_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461be59b3be5a31cd1c7c2_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bf3ba5f4569c02f5f10_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bff8c74816d8fb2d26d_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461c0ca20f659ce33312ee_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6c03f63139e8465a3bb9d4b4785_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_moved_flow_outcomes_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_moved_hygiene_pipeline_tag_value"
},
{
"parameterName": "pipeline_tag_value",
"queryName": "parameter_moved_dd_util_pipeline_tag_value"
}
]
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Pipeline Tags",
"description": "Filter pipelines by tag values"
}
}
},
"position": {
"x": 0,
"y": 0,
"width": 1,
"height": 2
}
}
],
"pageType": "PAGE_TYPE_GLOBAL_FILTERS"
},
{
"name": "75df3911",
"displayName": "Tables Metrics",
"layout": [
{
"widget": {
"name": "db12c0c1",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "3b8b7d86",
"fields": [
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "minutely(Error Time)",
"expression": "DATE_TRUNC(\"MINUTE\", `Error Time`)"
},
{
"name": "count(*)",
"expression": "COUNT(`*`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "minutely(Error Time)",
"scale": {
"type": "temporal"
},
"displayName": "Error Time"
},
"y": {
"fieldName": "count(*)",
"scale": {
"type": "quantitative"
},
"displayName": "Count of Records"
},
"color": {
"fieldName": "Target Table",
"scale": {
"type": "categorical"
},
"displayName": "Target Table"
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Table Errors",
"description": "Detected errors while processing tables"
}
}
},
"position": {
"x": 0,
"y": 2,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "841d82eb",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "8897fc81",
"fields": [
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "minutely(Warning Time)",
"expression": "DATE_TRUNC(\"MINUTE\", `Warning Time`)"
},
{
"name": "count(*)",
"expression": "COUNT(`*`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "minutely(Warning Time)",
"scale": {
"type": "categorical"
},
"displayName": "Warning Time"
},
"y": {
"fieldName": "count(*)",
"scale": {
"type": "quantitative"
},
"displayName": "Count of Records"
},
"color": {
"fieldName": "Target Table",
"scale": {
"type": "categorical"
},
"displayName": "Target Table"
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Table Warnings",
"description": "Detected warnings while processing tables"
}
}
},
"position": {
"x": 0,
"y": 23,
"width": 6,
"height": 5
}
},
{
"widget": {
"name": "6c9cec76",
"multilineTextboxSpec": {
"lines": ["# Errors and Warnings"]
}
},
"position": {
"x": 0,
"y": 0,
"width": 6,
"height": 2
}
},
{
"widget": {
"name": "e968324c",
"multilineTextboxSpec": {
"lines": ["# Rates and Throughput"]
}
},
"position": {
"x": 0,
"y": 46,
"width": 6,
"height": 2
}
},
{
"widget": {
"name": "edb32903",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "2660d018",
"fields": [
{
"name": "Flow",
"expression": "`Flow`"
},
{
"name": "hourly(Event Time)",
"expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
},
{
"name": "sum(Appended Rows)",
"expression": "SUM(`Appended Rows`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "hourly(Event Time)",
"scale": {
"type": "temporal"
}
},
"y": {
"fieldName": "sum(Appended Rows)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Flow",
"scale": {
"type": "categorical"
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Hourly Output Rows Throughput",
"description": "Number of output rows to the Flows per Hour"
}
}
},
"position": {
"x": 0,
"y": 48,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "c4b2aec5",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "3b8b7d86",
"fields": [
{
"name": "count(*)",
"expression": "COUNT(`*`)"
},
{
"name": "Error Code",
"expression": "`Error Code`"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "pie",
"encodings": {
"angle": {
"fieldName": "count(*)",
"scale": {
"type": "quantitative"
},
"displayName": "Number of occurrences"
},
"color": {
"fieldName": "Error Code",
"scale": {
"type": "categorical"
}
},
"label": {
"show": true
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Error Frequency",
"description": "Distribution of errors by error codes"
}
}
},
"position": {
"x": 0,
"y": 8,
"width": 6,
"height": 8
}
},
{
"widget": {
"name": "115fa69e",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "2660d018",
"fields": [
{
"name": "Flow",
"expression": "`Flow`"
},
{
"name": "hourly(Event Time)",
"expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
},
{
"name": "sum(Appended Bytes)",
"expression": "SUM(`Appended Bytes`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "hourly(Event Time)",
"scale": {
"type": "temporal"
}
},
"y": {
"fieldName": "sum(Appended Bytes)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Flow",
"scale": {
"type": "categorical"
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Hourly Appended Bytes Throughput",
"description": "Number of appended Bytes to the Flows per Hour"
}
}
},
"position": {
"x": 0,
"y": 54,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "afdff795",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "2660d018",
"fields": [
{
"name": "Flow",
"expression": "`Flow`"
},
{
"name": "hourly(Event Time)",
"expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
},
{
"name": "sum(Upserted Rows)",
"expression": "SUM(`Upserted Rows`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "hourly(Event Time)",
"scale": {
"type": "temporal"
}
},
"y": {
"fieldName": "sum(Upserted Rows)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Flow",
"scale": {
"type": "categorical"
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Hourly Upserted Rows Throughput",
"description": "Number of Upserted Rows (apply_changes flows) to the Flows per Hour"
}
}
},
"position": {
"x": 0,
"y": 60,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "05a978f3",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "2660d018",
"fields": [
{
"name": "Flow",
"expression": "`Flow`"
},
{
"name": "hourly(Event Time)",
"expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
},
{
"name": "sum(Deleted Rows)",
"expression": "SUM(`Deleted Rows`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "hourly(Event Time)",
"scale": {
"type": "temporal"
}
},
"y": {
"fieldName": "sum(Deleted Rows)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Flow",
"scale": {
"type": "categorical"
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Hourly Deleted Rows Throughput",
"description": "Number of Deleted Rows (apply_changes flows) to the Flows per Hour"
}
}
},
"position": {
"x": 0,
"y": 66,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "86b20f22",
"multilineTextboxSpec": {
"lines": ["# Latencies"]
}
},
"position": {
"x": 0,
"y": 72,
"width": 6,
"height": 2
}
},
{
"widget": {
"name": "20b6da4a",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "b72dbc41",
"fields": [
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "hourly(Event Time)",
"expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
},
{
"name": "sum(Backlog Rows)",
"expression": "SUM(`Backlog Rows`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "hourly(Event Time)",
"scale": {
"type": "temporal"
}
},
"y": {
"fieldName": "sum(Backlog Rows)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Target Table",
"scale": {
"type": "categorical"
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Hourly Backlogged Rows",
"description": "Number of backlogged rows to the Target Tables per Hour"
}
}
},
"position": {
"x": 0,
"y": 74,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "2828faa0",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "b72dbc41",
"fields": [
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "hourly(Event Time)",
"expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
},
{
"name": "sum(Backlog Bytes)",
"expression": "SUM(`Backlog Bytes`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "hourly(Event Time)",
"scale": {
"type": "temporal"
}
},
"y": {
"fieldName": "sum(Backlog Bytes)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Target Table",
"scale": {
"type": "categorical"
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Hourly Backlogged Bytes",
"description": "Number of backlogged bytes to the Target Tables per Hour"
}
}
},
"position": {
"x": 0,
"y": 80,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "bbe94522",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "b72dbc41",
"fields": [
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "hourly(Event Time)",
"expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
},
{
"name": "sum(Backlog Files)",
"expression": "SUM(`Backlog Files`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "hourly(Event Time)",
"scale": {
"type": "temporal"
}
},
"y": {
"fieldName": "sum(Backlog Files)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Target Table",
"scale": {
"type": "categorical"
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Hourly Backlogged Files",
"description": "Number of backlogged files to the Target Tables per Hour"
}
}
},
"position": {
"x": 0,
"y": 86,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "9d57fd5f",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "b72dbc41",
"fields": [
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "hourly(Event Time)",
"expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
},
{
"name": "max(Backlog Seconds)",
"expression": "MAX(`Backlog Seconds`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "hourly(Event Time)",
"scale": {
"type": "temporal"
}
},
"y": {
"fieldName": "max(Backlog Seconds)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Target Table",
"scale": {
"type": "categorical"
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Hourly Backlog Seconds",
"description": "Max of backlog seconds to the Target Tables per Hour"
}
}
},
"position": {
"x": 0,
"y": 92,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "dropped-rows-due-to-expectations-by-table",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "bc1ad993",
"fields": [
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "minutely(Event Time)",
"expression": "DATE_TRUNC(\"MINUTE\", `Event Time`)"
},
{
"name": "sum(Expectations Dropped Rows)",
"expression": "SUM(`Expectations Dropped Rows`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "minutely(Event Time)",
"scale": {
"type": "categorical"
}
},
"y": {
"fieldName": "sum(Expectations Dropped Rows)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Target Table",
"scale": {
"type": "categorical"
}
}
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Dropped rows",
"description": "Number of dropped rows due to expectation failures by table"
}
}
},
"position": {
"x": 0,
"y": 16,
"width": 6,
"height": 7
}
},
{
"widget": {
"name": "hourly-expectations-failures-by-expectation",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "e9270b22",
"fields": [
{
"name": "Expectation Name",
"expression": "`Expectation Name`"
},
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "minutely(Check Time)",
"expression": "DATE_TRUNC(\"MINUTE\", `Check Time`)"
},
{
"name": "sum(Rows Failed)",
"expression": "SUM(`Rows Failed`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "minutely(Check Time)",
"scale": {
"type": "temporal"
}
},
"y": {
"fieldName": "sum(Rows Failed)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Expectation Name",
"scale": {
"type": "categorical"
}
},
"extra": [
{
"fieldName": "Target Table"
}
]
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Expectations failures by expectation",
"description": "Number of rows that failed expectations"
}
}
},
"position": {
"x": 0,
"y": 28,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "548dd03c",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "e9270b22",
"fields": [
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "Expectation Name",
"expression": "`Expectation Name`"
},
{
"name": "minutely(Check Time)",
"expression": "DATE_TRUNC(\"MINUTE\", `Check Time`)"
},
{
"name": "sum(Rows Failed)",
"expression": "SUM(`Rows Failed`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "minutely(Check Time)",
"scale": {
"type": "temporal"
}
},
"y": {
"fieldName": "sum(Rows Failed)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Target Table",
"scale": {
"type": "categorical"
}
},
"extra": [
{
"fieldName": "Expectation Name"
}
]
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Expectations failures by table",
"description": "Number of rows that failed expectations"
}
}
},
"position": {
"x": 0,
"y": 40,
"width": 6,
"height": 6
}
},
{
"widget": {
"name": "a8b05c55",
"queries": [
{
"name": "main_query",
"query": {
"datasetName": "e9270b22",
"fields": [
{
"name": "Expectation Name",
"expression": "`Expectation Name`"
},
{
"name": "sum(Rows Failed)",
"expression": "SUM(`Rows Failed`)"
},
{
"name": "sum(Rows Passed)",
"expression": "SUM(`Rows Passed`)"
},
{
"name": "Target Table",
"expression": "`Target Table`"
},
{
"name": "minutely(Check Time)",
"expression": "DATE_TRUNC(\"MINUTE\", `Check Time`)"
},
{
"name": "avg(Failure Percentage)",
"expression": "AVG(`Failure Percentage`)"
}
],
"disaggregated": false
}
}
],
"spec": {
"version": 3,
"widgetType": "bar",
"encodings": {
"x": {
"fieldName": "minutely(Check Time)",
"scale": {
"type": "categorical"
}
},
"y": {
"fieldName": "avg(Failure Percentage)",
"scale": {
"type": "quantitative"
}
},
"color": {
"fieldName": "Expectation Name",
"scale": {
"type": "categorical"
}
},
"extra": [
{
"fieldName": "sum(Rows Failed)"
},
{
"fieldName": "sum(Rows Passed)"
},
{
"fieldName": "Target Table"
}
]
},
"frame": {
"showTitle": true,
"showDescription": true,
"title": "Expectations failure percentage by expectation",
"description": "Percentage of rows that failed expectations"
}
}
},
"position": {
"x": 0,
"y": 34,
"width": 6,
"height": 6
}
}
],
"pageType": "PAGE_TYPE_CANVAS"
}
],
"uiSettings": {
"theme": {
"widgetHeaderAlignment": "ALIGNMENT_UNSPECIFIED"
},
"applyModeEnabled": false
}
}