package errhandler import ( "testing" "github.com/stretchr/testify/require" "github.com/tidbcloud/dataflow-service/internal/db/entity" ) func TestPrettyMessage(t *testing.T) { msg := ` one two github.com/pingcap/tidb-lightning/lightning/mydump.(*mdLoaderSetup).setup /go/src/github.com/pingcap/tidb-lightning/lightning/mydum ` get := PrettyMessage(msg, uint32(entity.ImportDataFormatCsv)) require.Equal(t, "one\ntwo", get) } func TestSimplifyError(t *testing.T) { errs := []string{ "AccessDenied: The ciphertext refers to a customer master key that does not exist, does not exist in this region, or you are not allowed to access", "BucketRegionError: incorrect region, the bucket is not in 'ap-northeast-1' region at endpoint ''", "[Lightning:PreCheck:ErrPreCheckFailed]tidb-lightning pre-check failed: Region distribution is unbalanced, the ratio of the regions count of the store(8) with least regions(7) to the store(1) with most regions(95879) is 7.300868803387604e-05, but we expect it must not be less than 0.5", "[Lightning:PreCheck:ErrPreCheckFailed]tidb-lightning pre-check failed: table(s) [`gharchive_dev`.`github_events`] are not empty", "[Lightning:Restore:ErrSchemaNotExists]table `nft`.`nft` schema not found", "[Lightning:Restore:ErrEncodeKV]encode kv error in file NFT_Sales_History_2.csv:0 at offset 4: syntax error: cannot have consecutive fields without separator", "[Lightning:Restore:ErrUnknownColumns]unknown columns in header (id,name,volume,volume_usd,market_cap) for table nftHisto", "[Lightning:PreCheck:ErrCheckDataSource]check data source error: [Lightning:Restore:ErrUnknownColumns]unknown columns in header (c_95747439,aaa_3,1657221693126,q25jdq8dl5upmih2se7zw4dbctpr6uosxmt9hx8bchwuetmzy0fqscozruyfwanxlblqctsbq0xyp942gnt9wu2aawv56hxvgdifam4tlisqj0kw0kyd7j4pjvbsxfy64ipkmgfbokne120lwksrvjndj71g6wrhmtpjlgubh3q7sfxbr7htipe5j36kjwcp4iyezedcv5rxkwem0gpfzl0yjkvvafykixbyfa5aem6vr4qrbqjqq5ngyd7guthqycw5njilhtelrzvuw7o1kif9w6lojyv2u3erfar66zbxyvo8cmancnmn6hzg8xz1fqbgyjzqj9y2boliwfvkrpaloegy9qezslwnrgs3fpjnevcdxrwvpir7gqfvuibwwsk12nw0on61xe3vbwfg38zmamdkgy87bdycbrufraffoz4zk579icpq5f9rqlntdhuqfwwiam5pztrpabywdzzvhdig4npktbdmpbuyszmzhx0rqabst9ovffk3nuvr9wmd3zpeoj59tiz6qmzc0ejrqvuqqfelilokpsmzompeeowviixfryrrmoucilobtij8vxc6lz1eefhxowfwlc58dwkfrnx3ekxwxlececebgqryplem1pouj9gdfa95gpdq85m5mev8tat17ebfru628bh3e6e02ht4g6nnb8zxwvluccfawntu3kmwrsim96dbq7cawtdbzcdyzjysp8tsmm4fpkiv4bhushgcokh2btvferonl7rprqebtcraxrgjlngt5o0ptutwd8m41bh15fphpb9jemoemaanxzcattvfhnneqwdzx1c31nurcyisexsljzylcrqrkgm7uok4wkxvsa3hzz3jxkpmapj6sbwebwphh46t7jbzsxl1vb7vxqgmqz9phsvu30ch460vcdybnvkjmsrjgefloxmuclgd4f2ggyaskxaw1seezbman62vrfgdbifcjp) for table t", "checksum mismatched remote vs local => (checksum: 11392935869726273266 vs 16511259432307356781) (total_kvs: 2227130352 vs 2286517030) (total_bytes:169255050357 vs 190427391764)", "data-source-dir 's3://op-data-datastore-poc-prod/csv_latest_versions_20220222_uncompressed/?region=eu-central-1' doesn't exist or contains no files", "[Lightning:Config:ErrInvalidConfig]parse filter failed: at :1: missing table pattern", "[tikv:9002]TiKV server timeout", "cluster is deleted", "[Lightning:Common:ErrUnknown]unknown error: SharedConfigLoadError: failed to load config file, /.aws/config", "[Lightning:Storage:ErrEmptySourceDir]data-source-dir 's3://tidbcloud-samples-testimport/test_import/test?region=us-west-2' doesn't exist or contains no files", "[Lightning:Storage:ErrStorageUnknown]unknown storage error: AccessDenied: User: arn:aws:sts::385595570414:assumed-role/dbaas-shared-a225bc83-import/1656408111895559373 is not authorized to perform: sts:AssumeRole on resource: arn:aws:iam::385595570414:rol", "[Lightning:Config:ErrInvalidConfig]parse filter failed: at :1: missing table pattern", "[Lightning:Restore:ErrSchemaNotExists]table `db`.`test1` schema not found", "[Lightning:Loader:ErrInvalidSchemaFile]invalid database schema file, duplicated item - test_db.lists-schema.sql/test_db-schema-create.sql", "[Lighting:Restore:ErrChecksumMismatch]checksum mismatched remote vs local => (checksum: 13249482660652401355 vs 5396461279094460892) (total_kvs: 2 vs 4) (total_bytes:66 vs 132)", "[Lightning:Restore:ErrCreateSchema]create schema failed, table: `matrix`.`alerts_labels`, stmt: restore table schema: run create schema job failed: Error 1426: Too big precision 9 specified for column 'TIMESTAMP'. Maximum is 6.", "[Lightning:PreCheck:ErrCheckCSVHeader]check csv header error: syntax error: cannot have consecutive fields without separator", "[Lightning:Restore:ErrRestoreTable]restore table edw_bin.transactions failed: The available disk of TiKV (db-tikv-0.db-tikv-peer.tidb1379661944644004109.svc:20160) only left 435939180544, and capacity is 4362862139015", "[tikv:9005]Region is unavailable", "[Lightning:Restore:ErrEncodeKV]encode kv error in file tpcc.new_order.000000000.csv:0 at offset 7536640: read tcp 10.0.160.67:37728->52.92.225.184:443: read: connection reset by peer", "[Lightning:Restore:ErrRestoreTable]restore table sf100_starschema.lineorder failed: failed to send request to remote worker, http://load-data-worker-1705686528495372542-104--1-service.tidb-serverless.svc.cluster.local:19500/load_data?cluster_id=7158108932969803892&task_id=1705686528495372542-104--1, status 404 Not Found", "The number of conflict errors exceeds the threshold configured by `conflict.threshold`: '10000'", } expected := []string{ "AccessDenied: The ciphertext refers to a customer master key that does not exist, does not exist in this region, or you are not allowed to access", "BucketRegionError: bucket not in `ap-northeast-1`", "RegionUnbalanced: TiDB-Lightning pre-check failed: The region distribution is unbalanced. Please submit a ticket for assistance.", "PreCheckFailed: table(s) [`gharchive_dev`.`github_events`] are not empty", "SchemaNotExists: table `nft`.`nft` schema not found. Please create it manually in TiDB first before retrying.", "SyntaxError: NFT_Sales_History_2.csv:0 at offset 4: syntax error: cannot have consecutive fields without separator", "UnknownColumns: unknown columns in CSV file header (id,name,volume,...) for table 'nftHisto'. Please check CSV file header line and re-import data. If the CSV file has a header line, ensure column names of the header line match the target table. If the file doesn't have a header line, disable \"With Header\" in \"CSV Configuration\".", "UnknownColumns: unknown columns in CSV file header (c_95747439,aaa_3,1657221693126,...) for table 't'. Please check CSV file header line and re-import data. If the CSV file has a header line, ensure column names of the header line match the target table. If the file doesn't have a header line, disable \"With Header\" in \"CSV Configuration\".", "ChecksumMismatch: Duplicate row detected. To resolve the issue, either resolve conflicts in primary keys or unique keys in the source data, or ensure the primary key or unique key definition for the target table are correct, and modify them if necessary before reimporting.", "FileNotExists: data-source-dir 's3://op-data-datastore-poc-prod/csv_latest_versions_20220222_uncompressed/?region=eu-central-1' doesn't exist or contains no files", "InvalidTablePattern: parse filter failed: at :1: missing table pattern", "ClusterErr: Ensure the cluster is available, and retry import again.", "ClusterErr: Ensure the cluster is available, and retry import again.", "AWSConfigLoadError: failed to load config file, /.aws/config", "FileNotExists: data-source-dir 's3://tidbcloud-samples-testimport/test_import/test?region=us-west-2' doesn't exist or contains no files", "AccessDenied: User: arn:aws:sts::385595570414:assumed-role/dbaas-shared-a225bc83-import/1656408111895559373 is not authorized to perform: sts:AssumeRole on resource: arn:aws:iam::385595570414:rol", "InvalidTablePattern: parse filter failed: at :1: missing table pattern", "SchemaNotExists: table `db`.`test1` schema not found. Please create it manually in TiDB first before retrying.", "InvalidSchemaFile: invalid database schema file, duplicated item - test_db.lists-schema.sql/test_db-schema-create.sql. Please check it before retrying.", "ChecksumMismatch: Duplicate row detected. To resolve the issue, either resolve conflicts in primary keys or unique keys in the source data, or ensure the primary key or unique key definition for the target table are correct, and modify them if necessary before reimporting.", "CreateSchemaError: It is recommended to create the table in TiDB manually and try again later. Details: create schema failed, table: `matrix`.`alerts_labels`, stmt: restore table schema: run create schema job failed: Error 1426: Too big precision 9 specified for column 'TIMESTAMP'. Maximum is 6.", "SyntaxError: check csv header error: syntax error: cannot have consecutive fields without separator", "TikvLowSpace: Less than 10% of space is available in TiKV. Please clean up the space or increase the TiKV capacity and try again.", "ChecksumError: Data imported successfully, but an error occurred during checksum process ([tikv:9005]Region is unavailable). Contact support for verification plan for relevant tables.", "ReadSourceFileError: Failed to read source file. Error encountered while accessing S3 file.", "InternalError: An error occurred during import.", "TooManyConflict: More than '10000' conflicts detected. Please check your data before re-importing.", } for i, errMsg := range errs { require.Equal(t, expected[i], PrettyMessage(errMsg, uint32(entity.ImportDataFormatCsv))) } testcases := []struct { errMsg string dataFormat entity.DataFormat expected string }{ { errMsg: "[Lightning:Restore:ErrUnknownColumns]unknown columns in header (id,name,volume,volume_usd,market_cap) for table nftHisto", dataFormat: entity.ImportDataFormatCsv, expected: "UnknownColumns: unknown columns in CSV file header (id,name,volume,...) for table 'nftHisto'. Please check CSV file header line and re-import data. If the CSV file has a header line, ensure column names of the header line match the target table. If the file doesn't have a header line, disable \"With Header\" in \"CSV Configuration\".", }, { errMsg: "[Lightning:Restore:ErrUnknownColumns]unknown columns in header (_col_0,_col_1,_col_2,_col_3,_col_4,_col_5,_col_6,_col_7,_col_8,_col_9,_col_10,_col_11,_col_12,_col_13,_col_14,_col_15) for table LINEITEM", dataFormat: entity.ImportDataFormatParquet, expected: "UnknownColumns: The columns (_col_0,_col_1,_col_2,...) in the Parquet file header do not match the columns of the target table 'LINEITEM'. Please review the Parquet file header, ensure that the column names match those of the target table, and re-import the data.", }, } for _, tc := range testcases { require.Equal(t, tc.expected, PrettyMessage(tc.errMsg, uint32(tc.dataFormat))) } } func TestExtractUnknownColumnsForDisplay(t *testing.T) { testCases := []struct { CaseName string Message string ExpectedColumns []string ExpectedTableName string }{ { CaseName: "short list", Message: "[Lightning:Restore:ErrUnknownColumns]unknown columns in header (id,name,volume,volume_usd,market_cap) for table nftHisto", ExpectedColumns: []string{"id", "name", "volume", "volume_usd", "market_cap"}, ExpectedTableName: "nftHisto", }, { CaseName: "long list", Message: "[Lightning:PreCheck:ErrCheckDataSource]check data source error: [Lightning:Restore:ErrUnknownColumns]unknown columns in header (c_95747439,aaa_3,1657221693126,q25jdq8dl5upmih2se7zw4dbctpr6uosxmt9hx8bchwuetmzy0fqscozruyfwanxlblqctsbq0xyp942gnt9wu2aawv56hxvgdifam4tlisqj0kw0kyd7j4pjvbsxfy64ipkmgfbokne120lwksrvjndj71g6wrhmtpjlgubh3q7sfxbr7htipe5j36kjwcp4iyezedcv5rxkwem0gpfzl0yjkvvafykixbyfa5aem6vr4qrbqjqq5ngyd7guthqycw5njilhtelrzvuw7o1kif9w6lojyv2u3erfar66zbxyvo8cmancnmn6hzg8xz1fqbgyjzqj9y2boliwfvkrpaloegy9qezslwnrgs3fpjnevcdxrwvpir7gqfvuibwwsk12nw0on61xe3vbwfg38zmamdkgy87bdycbrufraffoz4zk579icpq5f9rqlntdhuqfwwiam5pztrpabywdzzvhdig4npktbdmpbuyszmzhx0rqabst9ovffk3nuvr9wmd3zpeoj59tiz6qmzc0ejrqvuqqfelilokpsmzompeeowviixfryrrmoucilobtij8vxc6lz1eefhxowfwlc58dwkfrnx3ekxwxlececebgqryplem1pouj9gdfa95gpdq85m5mev8tat17ebfru628bh3e6e02ht4g6nnb8zxwvluccfawntu3kmwrsim96dbq7cawtdbzcdyzjysp8tsmm4fpkiv4bhushgcokh2btvferonl7rprqebtcraxrgjlngt5o0ptutwd8m41bh15fphpb9jemoemaanxzcattvfhnneqwdzx1c31nurcyisexsljzylcrqrkgm7uok4wkxvsa3hzz3jxkpmapj6sbwebwphh46t7jbzsxl1vb7vxqgmqz9phsvu30ch460vcdybnvkjmsrjgefloxmuclgd4f2ggyaskxaw1seezbman62vrfgdbifcjp) for table t", ExpectedColumns: []string{"c_95747439", "aaa_3", "1657221693126", "..."}, ExpectedTableName: "t", }, { CaseName: "abnormal message", Message: "abcdefg", ExpectedColumns: nil, ExpectedTableName: "", }, { CaseName: "missing table name", Message: "[Lightning:Restore:ErrUnknownColumns]unknown columns in header (id,name,volume,volume_usd,market_cap) for table ", ExpectedColumns: []string{"id", "name", "volume", "volume_usd", "market_cap"}, ExpectedTableName: "", }, { CaseName: "missing columns", Message: "[Lightning:Restore:ErrUnknownColumns]unknown columns in header () for table t", ExpectedColumns: []string{}, ExpectedTableName: "t", }, { CaseName: "space column names", Message: "[Lightning:Restore:ErrUnknownColumns]unknown columns in header (id, name, volume, volume_usd, market_cap) for table t ", ExpectedColumns: []string{"id", "name", "volume", "volume_usd", "market_cap"}, ExpectedTableName: "t", }, } for _, tc := range testCases { t.Run(tc.CaseName, func(t *testing.T) { cols, tableName := extractUnknownColumnsForDisplay(tc.Message) require.Equal(t, tc.ExpectedColumns, cols) require.Equal(t, tc.ExpectedTableName, tableName) }) } }