{ "jobConfig": { "name": "prod-dataflow-svc-metadb-etl", "description": "", "role": "arn:aws:iam::380838443567:role/AWSGlueServiceRole-Prod-metadb-etl", "command": "glueetl", "version": "4.0", "runtime": null, "workerType": "G.1X", "numberOfWorkers": 2, "maxCapacity": 2, "maxRetries": 0, "timeout": 2880, "maxConcurrentRuns": 1, "security": "none", "scriptName": "prod-dataflow-svc-metadb-etl.py", "scriptLocation": "s3://aws-glue-assets-380838443567-us-west-2/scripts/", "language": "python-3", "jobParameters": [], "tags": [], "jobMode": "VISUAL_MODE", "createdOn": "2023-08-29T09:12:49.554Z", "developerMode": false, "connectionsList": [ "prod-dataflow-svc-conn" ], "temporaryDirectory": "s3://aws-glue-assets-380838443567-us-west-2/temporary/", "logging": true, "glueHiveMetastore": true, "etlAutoTuning": false, "metrics": true, "spark": true, "bookmark": "job-bookmark-disable", "sparkPath": "s3://aws-glue-assets-380838443567-us-west-2/sparkHistoryLogs/", "flexExecution": false, "minFlexWorkers": null }, "dag": { "node-1693356460515": { "nodeId": "node-1693356460515", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693302191834" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693356460515", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/dm_jobs/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 }, "node-1693301949357": { "nodeId": "node-1693301949357", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "df_jobs", "generatedNodeName": "df_jobs_node1693301949357", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "df_jobs", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693302071279": { "nodeId": "node-1693302071279", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "dm_job_modifications", "generatedNodeName": "dm_job_modifications_node1693302071279", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "dm_job_modifications", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693301218125": { "nodeId": "node-1693301218125", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693301204476" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693301218125", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693301204476", "alias": "backup_settings" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n id,\n project_id,\n tenant_id,\n cluster_id,\n created_at,\n updated_at,\n auto_backup,\n pi_tr \nFROM\n backup_settings;", "codeGenVersion": 2 }, "node-1693300533826": { "nodeId": "node-1693300533826", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693300521641" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693300533826", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/cdc_tasks/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 }, "node-1693302223428": { "nodeId": "node-1693302223428", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "dm_meterings", "generatedNodeName": "dm_meterings_node1693302223428", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "dm_meterings", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693302242808": { "nodeId": "node-1693302242808", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693302223428" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693302242808", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693302223428", "alias": "dm_meterings" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n `id`,\n `created_at`,\n `updated_at`,\n `kind`,\n `cost`,\n `job_id`,\n `cluster_id`,\n `tenant_id`,\n `project_id`,\n `start_ts`,\n `end_ts`,\n `cost_id`,\n `synced`\nFROM\n dm_meterings;", "codeGenVersion": 2 }, "node-1693301026634": { "nodeId": "node-1693301026634", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693300997204" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693301026634", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/backups/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 }, "node-1693356604986": { "nodeId": "node-1693356604986", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693301929118" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693356604986", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/df_conn_profiles/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 }, "node-1693302162665": { "nodeId": "node-1693302162665", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "dm_jobs", "generatedNodeName": "dm_jobs_node1693302162665", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "dm_jobs", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693301293704": { "nodeId": "node-1693301293704", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693301171005" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693301293704", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/log_backups/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 }, "node-1693302347242": { "nodeId": "node-1693302347242", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693302330510" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693302347242", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693302330510", "alias": "dm_prechecks" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n `id`,\n `created_at`,\n `updated_at`,\n `status`,\n `err_msg`,\n `total_item`,\n `full_instance_migrate`,\n `tenant_id`,\n `project_id`,\n `cluster_id`\nFROM\n dm_prechecks;", "codeGenVersion": 2 }, "node-1693356513899": { "nodeId": "node-1693356513899", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693302084083" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693356513899", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/dm_job_modifications/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 }, "node-1693300521641": { "nodeId": "node-1693300521641", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693300492409" ], "name": "Select From Collection", "generatedNodeName": "SelectFromCollection_node1693300521641", "classification": "Transform", "type": "SelectFromCollection", "index": 0, "parentsValid": true, "calculatedType": "", "codeGenVersion": 2 }, "node-1693301388333": { "nodeId": "node-1693301388333", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693301218125" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693301388333", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/backup_settings/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 }, "node-1693356367803": { "nodeId": "node-1693356367803", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693302347242" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693356367803", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/dm_prechecks/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 }, "node-1693301986110": { "nodeId": "node-1693301986110", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "df_private_link_endpoints", "generatedNodeName": "df_private_link_endpoints_node1693301986110", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "df_private_link_endpoints", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693301204476": { "nodeId": "node-1693301204476", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "backup_settings", "generatedNodeName": "backup_settings_node1693301204476", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "backup_settings", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693302309962": { "nodeId": "node-1693302309962", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693302295807" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693302309962", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693302295807", "alias": "dm_precheck_items" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n `id`,\n `created_at`,\n `updated_at`,\n `precheck_id`,\n `desc`,\n `status`,\n `err_msg`,\n `results`\nFROM\n dm_precheck_items;", "codeGenVersion": 2 }, "node-1693301838726": { "nodeId": "node-1693301838726", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "imports", "generatedNodeName": "imports_node1693301838726", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "imports", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693302121728": { "nodeId": "node-1693302121728", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "dm_job_sources", "generatedNodeName": "dm_job_sources_node1693302121728", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "dm_job_sources", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693302191834": { "nodeId": "node-1693302191834", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693302162665" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693302191834", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693302162665", "alias": "dm_jobs" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n `id`,\n `created_at`,\n `updated_at`,\n `deleted_at`,\n `tenant_id`,\n `project_id`,\n `cluster_id`,\n `cluster_user`,\n `mode`,\n `shard_mode`,\n `online_ddl`,\n `collation_compatible`,\n `spec_id`,\n `precheck_id`,\n `full_part_status_synced`\nFROM\n dm_jobs;", "codeGenVersion": 2 }, "node-1693301903141": { "nodeId": "node-1693301903141", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "df_conn_profiles", "generatedNodeName": "df_conn_profiles_node1693301903141", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "df_conn_profiles", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693356421285": { "nodeId": "node-1693356421285", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693302242808" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693356421285", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/dm_meterings/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 }, "node-1693302005788": { "nodeId": "node-1693302005788", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693301986110" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693302005788", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693301986110", "alias": "df_private_link_endpoints" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n `id`,\n `created_at`,\n `updated_at`,\n `deleted_at`,\n `creator_uid`,\n `tenant_id`,\n `project_id`,\n `provider`,\n `region`,\n `phase`,\n `endpoint_id`,\n `err_msg`\nFROM\n df_private_link_endpoints;", "codeGenVersion": 2 }, "node-1693356398922": { "nodeId": "node-1693356398922", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693302309962" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693356398922", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/dm_precheck_items/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 }, "node-1693301243034": { "nodeId": "node-1693301243034", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693301121450" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693301243034", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/backup_schedules/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 }, "node-1693300980953": { "nodeId": "node-1693300980953", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "backups", "generatedNodeName": "backups_node1693300980953", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "backups", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693302295807": { "nodeId": "node-1693302295807", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "dm_precheck_items", "generatedNodeName": "dm_precheck_items_node1693302295807", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "dm_precheck_items", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693356483967": { "nodeId": "node-1693356483967", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693302132429" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693356483967", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/dm_job_sources/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 }, "node-1693301929118": { "nodeId": "node-1693301929118", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693301903141" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693301929118", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693301903141", "alias": "df_conn_profiles" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n id,\n created_at,\n updated_at,\n tenant_id,\n project_id,\n cluster_id,\n creator_uid,\n name,\n type,\n version,\n region,\n conn_method,\n endpoint_service_name,\n host,\n port,\n `user`,\n same_region\nFROM\n df_conn_profiles;", "codeGenVersion": 2 }, "node-1693300428086": { "nodeId": "node-1693300428086", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "cdc_tasks", "generatedNodeName": "cdc_tasks_node1693300428086", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "cdc_tasks", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693302084083": { "nodeId": "node-1693302084083", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693302071279" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693302084083", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693302071279", "alias": "dm_job_modifications" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n `id`,\n `created_at`,\n `updated_at`,\n `job_id`,\n `type`,\n `param`,\n `finished_at`,\n `operator_id`\nFROM\n dm_job_modifications;", "codeGenVersion": 2 }, "node-1693300492409": { "nodeId": "node-1693300492409", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693300473561" ], "name": "Custom Transform", "generatedNodeName": "CustomTransform_node1693300492409", "classification": "Transform", "type": "CustomCode", "code": "from pyspark.sql.functions import col, udf\nfrom pyspark.sql.types import StringType\nimport json\n\n# 自定义 UDF 函数来处理 changefeed_param 字段\ndef process_changefeed_param(param):\n try:\n changefeed_param = json.loads(param)\n selected_data = {key: changefeed_param[key] for key in [\"connector_type\", \"changefeed_instance_type\"] if key in changefeed_param}\n return json.dumps(selected_data)\n except:\n return None\n\n# 创建 UDF\nprocess_changefeed_param_udf = udf(process_changefeed_param, StringType())\n\n# 转换 DynamicFrame 为 DataFrame\ndf = dfc.select(list(dfc.keys())[0]).toDF()\n\n# 使用 UDF 处理 changefeed_param 字段\ndf = df.withColumn(\"changefeed_param\", process_changefeed_param_udf(col(\"changefeed_param\")))\n\n# 将 DataFrame 转换回 DynamicFrame\ntransformed_dfc = DynamicFrame.fromDF(df, glueContext, \"transformed_dfc\")\ntransformed_dfc_col = DynamicFrameCollection({\"CustomTransform0\": transformed_dfc}, glueContext)\nreturn transformed_dfc_col\n", "className": "MyTransform", "outputSchemas": null, "parentsValid": true, "calculatedType": "", "codeGenVersion": 2 }, "node-1693301156685": { "nodeId": "node-1693301156685", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "log_backups", "generatedNodeName": "log_backups_node1693301156685", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "log_backups", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693301121450": { "nodeId": "node-1693301121450", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693301099766" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693301121450", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693301099766", "alias": "backup_schedules" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n id,\n created_at,\n updated_at,\n deleted_at,\n cluster_id,\n user_id,\n name,\n backup_cycle,\n SCHEDULE,\n max_backups,\n method,\n description,\n next_backup_time,\n retention_period,\n cluster_type,\n cleaned_up \nFROM\n backup_schedules;", "codeGenVersion": 2 }, "node-1693300473561": { "nodeId": "node-1693300473561", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693300428086" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693300473561", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693300428086", "alias": "cdc_tasks" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n id,\n user_id, \n project_id,\n tenant_id,\n cluster_id,\n created_at,\n updated_at,\n name,\n cluster_id,\n down_stream_type,\n status,\n start_time_stamp,\n target_time_stamp,\n error_message,\n ignore_replication,\n checkpoint_tso,\n retry,\n cdc_cluster_id,\n is_cleared,\n changefeed_param\nFROM\n cdc_tasks;", "codeGenVersion": 2 }, "node-1693356644815": { "nodeId": "node-1693356644815", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693301878957" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693356644815", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/imports/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 }, "node-1693301171005": { "nodeId": "node-1693301171005", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693301156685" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693301171005", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693301156685", "alias": "log_backups" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n id,\n project_id,\n tenant_id,\n cluster_id,\n created_at,\n updated_at,\n deleted_at,\n NAME,\n STORAGE,\n start_ts,\n checkpoint_ts,\n truncate_until,\n schedule_at,\n pending_at,\n stopping_at \nFROM\n log_backups;", "codeGenVersion": 2 }, "node-1693356553232": { "nodeId": "node-1693356553232", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693302005788" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693356553232", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/df_private_link_endpoints/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 }, "node-1693301965109": { "nodeId": "node-1693301965109", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693301949357" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693301965109", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693301949357", "alias": "df_jobs" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n id,\n created_at,\n updated_at,\n deleted_at,\n tenant_id,\n project_id,\n cluster_id,\n name,\n one_time,\n phase,\n service_code,\n creator_uid,\n region,\n provider,\n finished_at\nFROM\n df_jobs;", "codeGenVersion": 2 }, "node-1693300997204": { "nodeId": "node-1693300997204", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693300980953" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693300997204", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693300980953", "alias": "backups" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n id,\n project_id,\n tenant_id,\n cluster_id,\n created_at,\n updated_at,\n deleted_at,\n finished_at,\n backup_schedule_id,\n backup_size,\n targets,\n cluster_version,\n STATUS,\n cleaned_up,\n old_backup_name,\n start_time_stamp,\n cluster_snapshot_id,\nSTORAGE \nFROM\n backups;", "codeGenVersion": 2 }, "node-1693302330510": { "nodeId": "node-1693302330510", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "dm_prechecks", "generatedNodeName": "dm_prechecks_node1693302330510", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "dm_prechecks", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693302132429": { "nodeId": "node-1693302132429", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693302121728" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693302132429", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693302121728", "alias": "dm_job_sources" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n `job_id`,\n `conn_profile_id`,\n `gtid`,\n `case_sensitive`,\n `binlog_name`,\n `binlog_pos`,\n `binlog_gtid`,\n `route_rules`,\n `ba_rules`,\n `dump_status`,\n `load_status`,\n `deleted_at`\nFROM\n dm_job_sources;", "codeGenVersion": 2 }, "node-1693301878957": { "nodeId": "node-1693301878957", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693301838726" ], "name": "SQL Query", "generatedNodeName": "SQLQuery_node1693301878957", "classification": "Transform", "type": "SqlCode", "sqlAliases": [ { "from": "node-1693301838726", "alias": "imports" } ], "outputSchemas": null, "parentsValid": true, "calculatedType": "", "sqlQuery": "SELECT\n id,\n created_at,\n updated_at,\n cluster_id,\n status,\n message,\n started_at,\n finished_at,\n clean_up,\n bucket_region,\n data_format,\n data_source_type,\n custom_csv_format,\n total_size,\n total_files,\n source_url,\n current_tables,\n completed_tables,\n pending_tables,\n completed_percent,\n processed_data_size,\n total_tables_count,\n post_import_completed_percent,\n task_id,\n name,\n local_import_cfg,\n creator,\n project_id,\n tenant_id,\n target_table_infos\nFROM\n imports;", "codeGenVersion": 2 }, "node-1693301099766": { "nodeId": "node-1693301099766", "dataPreview": false, "previewAmount": 0, "inputs": [], "name": "backup_schedules", "generatedNodeName": "backup_schedules_node1693301099766", "classification": "DataSource", "type": "MySQL", "isCatalog": false, "database": "dbaas", "table": "backup_schedules", "connectionName": "prod-dataflow-svc-conn", "connectionType": "mysql", "redshiftTmpDir": "", "outputSchemas": null, "calculatedType": "", "codeGenVersion": 2 }, "node-1693356575482": { "nodeId": "node-1693356575482", "dataPreview": false, "previewAmount": 0, "inputs": [ "node-1693301965109" ], "name": "Amazon S3", "generatedNodeName": "AmazonS3_node1693356575482", "classification": "DataSink", "type": "S3", "streamingBatchInterval": 100, "format": "csv", "compression": "none", "path": "s3://centralized-storage-prod-pingcap/metadb/dataflow-svc/df_jobs/", "partitionKeys": [], "schemaChangePolicy": { "enableUpdateCatalog": false, "updateBehavior": null, "database": null, "table": null }, "updateCatalogOptions": "none", "calculatedType": "", "codeGenVersion": 2 } }, "hasBeenSaved": false }