Skip to content

fix: Row validations fail for values with trailing newline #1415

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Feb 4, 2025
Merged

fix: Row validations fail for values with trailing newline #1415

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
dcf6898
test: Add tests for data with special characters such as new line
nj1973 Jan 29, 2025
0f245ef
test: Add tests for data with special characters such as new line
nj1973 Jan 29, 2025
228dc1a
chore: dos2unix conversion on Spanner DDL file
nj1973 Jan 29, 2025
a3431d0
test: Add tests for data with special characters such as new line
nj1973 Jan 29, 2025
0c39a7f
Merge remote-tracking branch 'origin/develop' into 1411-row-validatio…
nj1973 Jan 31, 2025
ca7ddf5
fix: Align RStrip behaviour across SQL engines
nj1973 Jan 31, 2025
622d1c3
fix: Align RStrip behaviour across SQL engines
nj1973 Jan 31, 2025
34be4c1
test: Add tests for data with special characters such as new line
nj1973 Jan 31, 2025
29d3635
fix: Align RStrip behaviour across SQL engines
nj1973 Jan 31, 2025
77cab76
test: Add tests for data with special characters such as new line
nj1973 Jan 31, 2025
e2a8645
Merge branch 'develop' into 1411-row-validations-fail-for-values-with…
nj1973 Feb 4, 2025
baca7a6
Update tests/resources/snowflake_test_tables.sql
nj1973 Feb 4, 2025
ce55b5c
Update tests/resources/sqlserver_test_tables.sql
nj1973 Feb 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion tests/resources/bigquery_test_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -655,7 +655,18 @@ CREATE TABLE `pso_data_validator`.`dvt_tricky_dates` (
, col_dt_high DATE
, col_ts_low DATETIME
, col_ts_epoch DATETIME
, col_ts_high DATETIME);
, col_ts_high DATETIME
) OPTIONS (description='Integration test table used to test potentially difficult Timestamps.');
INSERT INTO `pso_data_validator`.`dvt_tricky_dates` VALUES
(1,DATE'1000-01-01',DATE'1970-01-01',DATE'9999-12-31'
,DATETIME'1000-01-01 00:00:00',DATETIME'1970-01-01 00:00:00',DATETIME'9999-12-31 23:59:59');

CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_tricky_strings` (
id INT64
, col_string STRING
, col_comment STRING
) OPTIONS (description='Integration test table used to test potentially difficult Strings.');
INSERT INTO `pso_data_validator`.`dvt_tricky_strings` VALUES
(1,'str\nstr','Contains: new line'), (2,'str\n','Trailing: new line'),
(3,'str\rstr','Contains: carriage return'), (4,'str\r','Trailing: carriage return'),
(5,'str\tstr','Contains: tab'), (6,'str\t','Trailing: tab');
5 changes: 3 additions & 2 deletions tests/resources/mysql_test_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -554,15 +554,16 @@ CREATE TABLE `pso_data_validator`.`dvt_many_cols`
) COMMENT 'Integration test table used to test validating many columns.';
INSERT INTO `pso_data_validator`.`dvt_many_cols` (id) values (1);

DROP TABLE `pso_data_validator`.`dvt_tricky_dates`;
DROP TABLE IF EXISTS `pso_data_validator`.`dvt_tricky_dates`;
CREATE TABLE `pso_data_validator`.`dvt_tricky_dates` (
id integer NOT NULL PRIMARY KEY
, col_dt_low date
, col_dt_epoch date
, col_dt_high date
, col_ts_low datetime(0)
, col_ts_epoch datetime(0)
, col_ts_high datetime(0));
, col_ts_high datetime(0)
) COMMENT='Integration test table used to test potentially difficult Timestamps.';
SET time_zone = '+00:00';
INSERT INTO `pso_data_validator`.`dvt_tricky_dates` VALUES
(1,'1000-01-01','1970-01-01','9999-12-31'
Expand Down
15 changes: 15 additions & 0 deletions tests/resources/oracle_test_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -764,7 +764,22 @@ CREATE TABLE pso_data_validator.dvt_tricky_dates (
, col_ts_low TIMESTAMP(0)
, col_ts_epoch TIMESTAMP(0)
, col_ts_high TIMESTAMP(0));
COMMENT ON TABLE pso_data_validator.dvt_tricky_dates IS 'Integration test table used to test potentially difficult Timestamps.';
INSERT INTO pso_data_validator.dvt_tricky_dates VALUES
(1,DATE'1000-01-01',DATE'1970-01-01',DATE'9999-12-31'
,TIMESTAMP'1000-01-01 00:00:00',TIMESTAMP'1970-01-01 00:00:00',TIMESTAMP'9999-12-31 23:59:59');
COMMIT;

DROP TABLE pso_data_validator.dvt_tricky_strings;
CREATE TABLE pso_data_validator.dvt_tricky_strings (
id NUMBER(5) NOT NULL PRIMARY KEY
, col_string VARCHAR2(20)
, col_comment VARCHAR2(40));
COMMENT ON TABLE pso_data_validator.dvt_tricky_strings IS 'Integration test table used to test potentially difficult Strings.';
INSERT INTO pso_data_validator.dvt_tricky_strings VALUES (1,'str'||CHR(10)||'str','Contains: new line');
INSERT INTO pso_data_validator.dvt_tricky_strings VALUES (2,'str'||CHR(10),'Trailing: new line');
INSERT INTO pso_data_validator.dvt_tricky_strings VALUES (3,'str'||CHR(13)||'str','Contains: carriage return');
INSERT INTO pso_data_validator.dvt_tricky_strings VALUES (4,'str'||CHR(13),'Trailing: carriage return');
INSERT INTO pso_data_validator.dvt_tricky_strings VALUES (5,'str'||CHR(9)||'str','Contains: tab');
INSERT INTO pso_data_validator.dvt_tricky_strings VALUES (6,'str'||CHR(9),'Trailing: tab');
COMMIT;
12 changes: 12 additions & 0 deletions tests/resources/postgresql_test_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -846,7 +846,19 @@ CREATE TABLE pso_data_validator.dvt_tricky_dates (
, col_ts_low timestamp(0)
, col_ts_epoch timestamp(0)
, col_ts_high timestamp(0));
COMMENT ON TABLE pso_data_validator.dvt_tricky_dates IS 'Integration test table used to test potentially difficult Timestamps.';
INSERT INTO pso_data_validator.dvt_tricky_dates VALUES
(1,DATE'1000-01-01',DATE'1970-01-01',DATE'9999-12-31'
,TIMESTAMP'1000-01-01 00:00:00',TIMESTAMP'1970-01-01 00:00:00',TIMESTAMP'9999-12-31 23:59:59');

DROP TABLE IF EXISTS pso_data_validator.dvt_tricky_strings;
CREATE TABLE pso_data_validator.dvt_tricky_strings (
id integer NOT NULL PRIMARY KEY
, col_string varchar(20)
, col_comment varchar(40));
COMMENT ON TABLE pso_data_validator.dvt_tricky_strings IS 'Integration test table used to test potentially difficult Strings.';
INSERT INTO pso_data_validator.dvt_tricky_strings VALUES
(1,E'str\nstr','Contains: new line'), (2,E'str\n','Trailing: new line'),
(3,E'str\rstr','Contains: carriage return'), (4,E'str\r','Trailing: carriage return'),
(5,E'str\tstr','Contains: tab'), (6,E'str\t','Trailing: tab');

15 changes: 15 additions & 0 deletions tests/resources/snowflake_test_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -627,6 +627,21 @@ CREATE TABLE PSO_DATA_VALIDATOR.PUBLIC.DVT_TRICKY_DATES (
, col_ts_low TIMESTAMP(0)
, col_ts_epoch TIMESTAMP(0)
, col_ts_high TIMESTAMP(0));
COMMENT ON TABLE PSO_DATA_VALIDATOR.PUBLIC.dvt_tricky_dates IS 'Integration test table used to test potentially difficult Timestamps.';
INSERT INTO PSO_DATA_VALIDATOR.PUBLIC.DVT_TRICKY_DATES VALUES
(1,DATE'1000-01-01',DATE'1970-01-01',DATE'9999-12-31'
,TIMESTAMP'1000-01-01 00:00:00',TIMESTAMP'1970-01-01 00:00:00',TIMESTAMP'9999-12-31 23:59:59');

DROP TABLE PSO_DATA_VALIDATOR.PUBLIC.dvt_tricky_strings;
CREATE TABLE PSO_DATA_VALIDATOR.PUBLIC.dvt_tricky_strings (
id NUMBER(5) NOT NULL PRIMARY KEY
, col_string VARCHAR(20)
, col_comment VARCHAR(40));
COMMENT ON TABLE PSO_DATA_VALIDATOR.PUBLIC.dvt_tricky_strings IS 'Integration test table used to test potentially difficult Strings.';
INSERT INTO PSO_DATA_VALIDATOR.PUBLIC.dvt_tricky_strings
SELECT 1,'str'||CHR(10)||'str','Contains: new line' UNION ALL
SELECT 2,'str'||CHR(10),'Trailing: new line' UNION ALL
SELECT 3,'str'||CHR(13)||'str','Contains: carriage return' UNION ALL
SELECT 4,'str'||CHR(13),'Trailing: carriage return' UNION ALL
SELECT 5,'str'||CHR(9)||'str','Contains: tab' UNION ALL
SELECT 6,'str'||CHR(9),'Trailing: tab';
12 changes: 12 additions & 0 deletions tests/resources/sqlserver_test_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -588,3 +588,15 @@ CREATE TABLE pso_data_validator.dvt_tricky_dates (
INSERT INTO pso_data_validator.dvt_tricky_dates VALUES
(1,'1000-01-01','1970-01-01','9999-12-31'
,'1000-01-01 00:00:00','1970-01-01 00:00:00','9999-12-31 23:59:59');

DROP TABLE pso_data_validator.dvt_tricky_strings;
CREATE TABLE pso_data_validator.dvt_tricky_strings (
id integer NOT NULL PRIMARY KEY
, col_string varchar(20)
, col_comment varchar(40));
INSERT INTO pso_data_validator.dvt_tricky_strings VALUES (1,'str'+CHAR(10)+'str','Contains: new line');
INSERT INTO pso_data_validator.dvt_tricky_strings VALUES (2,'str'+CHAR(10),'Trailing: new line');
INSERT INTO pso_data_validator.dvt_tricky_strings VALUES (3,'str'+CHAR(13)+'str','Contains: carriage return');
INSERT INTO pso_data_validator.dvt_tricky_strings VALUES (4,'str'+CHAR(13),'Trailing: carriage return');
INSERT INTO pso_data_validator.dvt_tricky_strings VALUES (5,'str'+CHAR(9)+'str','Contains: tab');
INSERT INTO pso_data_validator.dvt_tricky_strings VALUES (6,'str'+CHAR(9),'Trailing: tab');
13 changes: 13 additions & 0 deletions tests/resources/teradata_test_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -695,3 +695,16 @@ INSERT INTO udf.dvt_tricky_dates VALUES
,TIMESTAMP'1000-01-01 00:00:00',TIMESTAMP'1970-01-01 00:00:00',TIMESTAMP'9999-12-31 23:59:59+00:00');
-- col_ts_high value above forced to UTC based on article below, but we still get wrong answer from the test:
-- https://support.teradata.com/knowledge?id=kb_article_view&sys_kb_id=0e81918ac36da9103eb2d88f05013138

DROP TABLE udf.dvt_tricky_strings;
CREATE TABLE udf.dvt_tricky_strings (
id INTEGER NOT NULL PRIMARY KEY
, col_string VARCHAR(20)
, col_comment VARCHAR(40));
COMMENT ON TABLE udf.dvt_tricky_strings IS 'Integration test table used to test potentially difficult Strings.';
INSERT INTO udf.dvt_tricky_strings VALUES (1,'str'||CHR(10)||'str','Contains: new line');
INSERT INTO udf.dvt_tricky_strings VALUES (2,'str'||CHR(10),'Trailing: new line');
INSERT INTO udf.dvt_tricky_strings VALUES (3,'str'||CHR(13)||'str','Contains: carriage return');
INSERT INTO udf.dvt_tricky_strings VALUES (4,'str'||CHR(13),'Trailing: carriage return');
INSERT INTO udf.dvt_tricky_strings VALUES (5,'str'||CHR(9)||'str','Contains: tab');
INSERT INTO udf.dvt_tricky_strings VALUES (6,'str'||CHR(9),'Trailing: tab');
13 changes: 13 additions & 0 deletions tests/system/data_sources/test_oracle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -943,6 +943,19 @@ def test_row_validation_tricky_dates_to_bigquery():
)


@mock.patch(
"data_validation.state_manager.StateManager.get_connection_config",
new=mock_get_connection_config,
)
def test_row_validation_tricky_strings_to_bigquery():
"""Test with string values containing special characters."""
row_validation_test(
tables="pso_data_validator.dvt_tricky_strings",
tc="bq-conn",
hash="*",
)


@mock.patch(
"data_validation.state_manager.StateManager.get_connection_config",
new=mock_get_connection_config,
Expand Down
13 changes: 13 additions & 0 deletions tests/system/data_sources/test_postgres.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1057,6 +1057,19 @@ def test_row_validation_tricky_dates_to_bigquery():
)


@mock.patch(
"data_validation.state_manager.StateManager.get_connection_config",
new=mock_get_connection_config,
)
def test_row_validation_tricky_strings_to_bigquery():
"""Test with string values containing special characters."""
row_validation_test(
tables="pso_data_validator.dvt_tricky_strings",
tc="bq-conn",
hash="*",
)


@mock.patch(
"data_validation.state_manager.StateManager.get_connection_config",
new=mock_get_connection_config,
Expand Down
13 changes: 13 additions & 0 deletions tests/system/data_sources/test_snowflake.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,19 @@ def test_row_validation_tricky_dates_to_bigquery():
)


@mock.patch(
"data_validation.state_manager.StateManager.get_connection_config",
new=mock_get_connection_config,
)
def test_row_validation_tricky_strings_to_bigquery():
"""Test with string values containing special characters."""
row_validation_test(
tables="PSO_DATA_VALIDATOR.PUBLIC.DVT_TRICKY_STRINGS=pso_data_validator.dvt_tricky_strings",
tc="bq-conn",
hash="*",
)


@mock.patch(
"data_validation.state_manager.StateManager.get_connection_config",
new=mock_get_connection_config,
Expand Down
13 changes: 13 additions & 0 deletions tests/system/data_sources/test_spanner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,19 @@ def test_row_validation_many_columns():
row_validation_many_columns_test()


@mock.patch(
"data_validation.state_manager.StateManager.get_connection_config",
new=mock_get_connection_config,
)
def test_row_validation_tricky_strings_to_bigquery():
"""Test with string values containing special characters."""
row_validation_test(
tables="pso_data_validator.dvt_tricky_strings",
tc="bq-conn",
hash="*",
)


@mock.patch(
"data_validation.state_manager.StateManager.get_connection_config",
new=mock_get_connection_config,
Expand Down
16 changes: 16 additions & 0 deletions tests/system/data_sources/test_sql_server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,22 @@ def test_row_validation_tricky_dates_to_bigquery():
)


@mock.patch(
"data_validation.state_manager.StateManager.get_connection_config",
new=mock_get_connection_config,
)
def test_row_validation_tricky_strings_to_bigquery():
"""Test with string values containing special characters."""
pytest.skip(
"Skipping test_row_validation_tricky_dates_to_bigquery because the version of SQL Server we have does not support rtrim of all whitespace."
)
row_validation_test(
tables="pso_data_validator.dvt_tricky_strings",
tc="bq-conn",
hash="*",
)


@mock.patch(
"data_validation.state_manager.StateManager.get_connection_config",
new=mock_get_connection_config,
Expand Down
13 changes: 13 additions & 0 deletions tests/system/data_sources/test_teradata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -803,6 +803,19 @@ def test_row_validation_tricky_dates_to_bigquery():
)


@mock.patch(
"data_validation.state_manager.StateManager.get_connection_config",
new=mock_get_connection_config,
)
def test_row_validation_tricky_strings_to_bigquery():
"""Test with string values containing special characters."""
row_validation_test(
tables="udf.dvt_tricky_strings=pso_data_validator.dvt_tricky_strings",
tc="bq-conn",
hash="*",
)


@mock.patch(
"data_validation.state_manager.StateManager.get_connection_config",
new=mock_get_connection_config,
Expand Down
Loading