Skip to content

Commit

Permalink
Use string columns instead of text columns.
Browse files Browse the repository at this point in the history
  • Loading branch information
@piotrczarnas
piotrczarnas committed Oct 19, 2024
1 parent b5fdbd3 commit a389811
Show file tree
Hide file tree
Showing 58 changed files with 1,467 additions and 1,429 deletions.
8 changes: 4 additions & 4 deletions dqops/src/main/java/com/dqops/cli/commands/column/impl/ColumnCliServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,10 +111,10 @@ public CliOperationStatus loadColumns(String connectionName,

Table resultTable = Table.create().addColumns(
LongColumn.create("Id"),
TextColumn.create("Connection name"),
TextColumn.create("Table name"),
TextColumn.create("Column name"),
TextColumn.create("Column type"),
StringColumn.create("Connection name"),
StringColumn.create("Table name"),
StringColumn.create("Column name"),
StringColumn.create("Column type"),
BooleanColumn.create("Disabled"));

for (ColumnSpec columnSpec: columnSpecs) {
Expand Down
16 changes: 8 additions & 8 deletions dqops/src/main/java/com/dqops/cli/commands/connection/impl/ConnectionCliServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,10 +151,10 @@ public CliOperationStatus showTableForConnection(String connectionName, String f
ColumnSpecMap columnSpecMap = tableSpec.getColumns();

Table resultTable = Table.create().addColumns(
TextColumn.create("Source column name"),
TextColumn.create("Source column type"),
TextColumn.create("Imported column name"),
TextColumn.create("Imported column type"));
StringColumn.create("Source column name"),
StringColumn.create("Source column type"),
StringColumn.create("Imported column name"),
StringColumn.create("Imported column type"));

for (Map.Entry<String, ColumnSpec> entry : columnSpecMap.entrySet()) {
String columnName = entry.getKey();
Expand Down Expand Up @@ -243,8 +243,8 @@ public CliOperationStatus loadTableList(String connectionName,
Collection<TableWrapper> tableWrappers = hierarchyNodeTreeSearcher.findTables(userHome, tableSearchFilters);

Table resultTable = Table.create().addColumns(
TextColumn.create("Source table name"),
TextColumn.create("Is imported"),
StringColumn.create("Source table name"),
StringColumn.create("Is imported"),
IntColumn.create("Imported columns count"),
IntColumn.create("Table hash"));

Expand Down Expand Up @@ -332,8 +332,8 @@ public CliOperationStatus loadSchemaList(String connectionName, TabularOutputFor
Collection<TableWrapper> tableWrappers = hierarchyNodeTreeSearcher.findTables(userHome, tableSearchFilters);

Table resultTable = Table.create().addColumns(
TextColumn.create("Source schema name"),
TextColumn.create("Is imported"),
StringColumn.create("Source schema name"),
StringColumn.create("Is imported"),
IntColumn.create("Imported tables count"));

for( SourceSchemaModel sourceSchemaModel : schemaModels) {
Expand Down
2 changes: 1 addition & 1 deletion dqops/src/main/java/com/dqops/cli/commands/data/DataStoragePerformanceCliCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ private void runOneIteration(LocalDate monthDate, Table newRows) {
}

private void generateRows(LocalDate monthDate, Table targetTable) {
TextColumn idColumn = targetTable.textColumn(CheckResultsColumnNames.ID_COLUMN_NAME);
StringColumn idColumn = targetTable.stringColumn(CheckResultsColumnNames.ID_COLUMN_NAME);
DoubleColumn actualValueColumn = targetTable.doubleColumn(CheckResultsColumnNames.ACTUAL_VALUE_COLUMN_NAME);
DateTimeColumn timePeriodColumn = targetTable.dateTimeColumn(CheckResultsColumnNames.TIME_PERIOD_COLUMN_NAME);

Expand Down
6 changes: 3 additions & 3 deletions dqops/src/main/java/com/dqops/cli/commands/data/impl/DataCliServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,9 @@ protected Table dataDeleteResultAsTable(DeleteStoredDataResult deleteStoredDataR
Iterable<ParquetPartitionId> partitionIds = partitionMap.keySet().stream().sorted().collect(Collectors.toList());

Table resultTable = Table.create().addColumns(
TextColumn.create("Data type"),
TextColumn.create("Connection"),
TextColumn.create("Table"),
StringColumn.create("Data type"),
StringColumn.create("Connection"),
StringColumn.create("Table"),
DateColumn.create("Month"),
IntColumn.create("Affected rows"),
BooleanColumn.create("Partition deleted"));
Expand Down
8 changes: 4 additions & 4 deletions dqops/src/main/java/com/dqops/cli/commands/table/impl/TableCliServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ public Table loadSchemaList(String connectionName, String schemaFilter) throws T
List<SourceSchemaModel> schemas = sourceConnection.listSchemas().stream()
.filter(schema -> (schemaFilter == null || StringPatternComparer.matchSearchPattern(schema.getSchemaName(), schemaFilter)))
.collect(Collectors.toList());
Table resultTable = Table.create().addColumns(TextColumn.create("Schema name"));
Table resultTable = Table.create().addColumns(StringColumn.create("Schema name"));
for (SourceSchemaModel schemaModel : schemas) {
Row row = resultTable.appendRow();
row.setString(0, schemaModel.getSchemaName());
Expand Down Expand Up @@ -189,9 +189,9 @@ public CliOperationStatus importTables(String connectionName, String schemaName,
public Table createTablesTableFromTableSpecList(Collection<TableWrapper> sourceTableWrappers, UserHome userHome) {
Table resultTable = Table.create().addColumns(
LongColumn.create("Id"),
TextColumn.create("Connection name"),
TextColumn.create("Schema name"),
TextColumn.create("Table name"),
StringColumn.create("Connection name"),
StringColumn.create("Schema name"),
StringColumn.create("Table name"),
IntColumn.create("Column count"));

for( TableWrapper sourceTableWrapper : sourceTableWrappers) {
Expand Down
10 changes: 5 additions & 5 deletions dqops/src/main/java/com/dqops/cli/terminal/TerminalTableWritterImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
import org.springframework.shell.table.TableModel;
import org.springframework.stereotype.Component;
import tech.tablesaw.api.Row;
import tech.tablesaw.api.StringColumn;
import tech.tablesaw.api.Table;
import tech.tablesaw.api.TextColumn;


/**
Expand Down Expand Up @@ -205,7 +205,7 @@ public void writeTable(Table table, boolean addBorder, boolean noHeader) {
@Override
public void writeTable(TableModel tableModel, boolean addBorder, boolean noHeader) {
int tableStartIndex = noHeader ? 0 : 1;
TextColumn[] headers = retrieveHeaders(tableModel);
StringColumn[] headers = retrieveHeaders(tableModel);
Table resultTable = Table.create().addColumns(headers);

for(int y = tableStartIndex; y < tableModel.getRowCount(); y++) {
Expand Down Expand Up @@ -324,12 +324,12 @@ else if (response == 'y' || response == 'Y') {
return false;
}

private TextColumn[] retrieveHeaders(TableModel tableModel) {
private StringColumn[] retrieveHeaders(TableModel tableModel) {
int columnCount = tableModel.getColumnCount();
TextColumn[] result = new TextColumn[columnCount];
StringColumn[] result = new StringColumn[columnCount];

for (int i = 0; i < columnCount; i++) {
result[i] = TextColumn.create(tableModel.getValue(0, i).toString());
result[i] = StringColumn.create(tableModel.getValue(0, i).toString());
}

return result;
Expand Down
2 changes: 1 addition & 1 deletion dqops/src/main/java/com/dqops/connectors/bigquery/BigQuerySqlRunnerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,7 @@ public List<Column<?>> createColumnsFromBigQuerySchema(Schema tableSchema) {
case STRUCT:
case ARRAY:
case GEOGRAPHY:
columns.add(tech.tablesaw.api.TextColumn.create(field.getName()));
columns.add(tech.tablesaw.api.StringColumn.create(field.getName()));
break;
case TIMESTAMP:
columns.add(tech.tablesaw.api.InstantColumn.create(field.getName()));
Expand Down
22 changes: 11 additions & 11 deletions dqops/src/main/java/com/dqops/core/incidents/IncidentImportQueueServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,7 @@ public List<IncidentNotificationMessage> importBatch(TableIncidentImportBatch ne
InstantColumn executedAtColumn = newCheckResults.instantColumn(CheckResultsColumnNames.EXECUTED_AT_COLUMN_NAME);
DateTimeColumn timePeriodColumn = newCheckResults.dateTimeColumn(CheckResultsColumnNames.TIME_PERIOD_COLUMN_NAME);
LongColumn checkResultIncidentHashColumn = newCheckResults.longColumn(CheckResultsColumnNames.INCIDENT_HASH_COLUMN_NAME);
TextColumn checkTypeColumn = newCheckResults.textColumn(CheckResultsColumnNames.CHECK_TYPE_COLUMN_NAME);
StringColumn checkTypeColumn = newCheckResults.stringColumn(CheckResultsColumnNames.CHECK_TYPE_COLUMN_NAME);
Selection selectionOfMonitoringCheckResults = checkTypeColumn.isEqualTo(CheckType.monitoring.getDisplayName());
Selection selectionOfPartitionedCheckResults = checkTypeColumn.isEqualTo(CheckType.partitioned.getDisplayName());
Selection selectionOfIssuesAboveMinSeverity = severityColumn.isGreaterThanOrEqualTo(minimumSeverityLevel);
Expand Down Expand Up @@ -403,7 +403,7 @@ public List<IncidentNotificationMessage> importBatch(TableIncidentImportBatch ne
IntColumn highestSeverityNewIncidentsColumn = this.allNewIncidentRows.intColumn(IncidentsColumnNames.HIGHEST_SEVERITY_COLUMN_NAME);
InstantColumn lastSeenNewIncidentsColumn = this.allNewIncidentRows.instantColumn(IncidentsColumnNames.LAST_SEEN_COLUMN_NAME);
InstantColumn incidentUntilNewIncidentsColumn = this.allNewIncidentRows.instantColumn(IncidentsColumnNames.INCIDENT_UNTIL_COLUMN_NAME);
TextColumn statusNewIncidentsColumn = this.allNewIncidentRows.textColumn(IncidentsColumnNames.STATUS_COLUMN_NAME);
StringColumn statusNewIncidentsColumn = this.allNewIncidentRows.stringColumn(IncidentsColumnNames.STATUS_COLUMN_NAME);

int[] issuesRowIndexes = selectionOfAllNewAlerts.toArray();
for (int i = 0; i < issuesRowIndexes.length; i++) {
Expand Down Expand Up @@ -462,7 +462,7 @@ public List<IncidentNotificationMessage> importBatch(TableIncidentImportBatch ne
int existingIncidentRowIndex = rowIndexesOfOldIncidents.getInt(ri);
InstantColumn existingIncidentsFirstSeenColumn = this.allExistingIncidentRows.instantColumn(IncidentsColumnNames.FIRST_SEEN_COLUMN_NAME);
InstantColumn existingIncidentsIncidentUntilColumn = this.allExistingIncidentRows.instantColumn(IncidentsColumnNames.INCIDENT_UNTIL_COLUMN_NAME);
TextColumn existingIncidentsStatusColumn = this.allExistingIncidentRows.textColumn(IncidentsColumnNames.STATUS_COLUMN_NAME);
StringColumn existingIncidentsStatusColumn = this.allExistingIncidentRows.stringColumn(IncidentsColumnNames.STATUS_COLUMN_NAME);

if (!executedAt.isBefore(existingIncidentsFirstSeenColumn.get(existingIncidentRowIndex)) &&
!executedAt.isAfter(existingIncidentsIncidentUntilColumn.get(existingIncidentRowIndex)) &&
Expand Down Expand Up @@ -648,12 +648,12 @@ public IncidentNotificationMessage changeIncidentStatus(IncidentStatusChangePara
}

String newStatusString = incidentStatusChangeParameters.getNewIncidentStatus().name();
Selection newRowsIncidentIdSelection = this.allNewIncidentRows.textColumn(IncidentsColumnNames.ID_COLUMN_NAME)
Selection newRowsIncidentIdSelection = this.allNewIncidentRows.stringColumn(IncidentsColumnNames.ID_COLUMN_NAME)
.isEqualTo(incidentStatusChangeParameters.getIncidentId());
if (!newRowsIncidentIdSelection.isEmpty()) {
// the updated row has other updates awaiting
int newRowIndex = newRowsIncidentIdSelection.get(0);
TextColumn newRowStatusColumn = this.allNewIncidentRows.textColumn(IncidentsColumnNames.STATUS_COLUMN_NAME);
StringColumn newRowStatusColumn = this.allNewIncidentRows.stringColumn(IncidentsColumnNames.STATUS_COLUMN_NAME);
String currentStatus = newRowStatusColumn.get(newRowIndex);
if (!Objects.equals(currentStatus, newStatusString)) {
newRowStatusColumn.set(newRowIndex, newStatusString);
Expand All @@ -666,7 +666,7 @@ public IncidentNotificationMessage changeIncidentStatus(IncidentStatusChangePara
return IncidentNotificationMessage.fromIncidentRow(messageParameters);
}
} else if (this.allExistingIncidentRows != null) {
Selection existingRowsIncidentIdSelection = this.allExistingIncidentRows.textColumn(IncidentsColumnNames.ID_COLUMN_NAME)
Selection existingRowsIncidentIdSelection = this.allExistingIncidentRows.stringColumn(IncidentsColumnNames.ID_COLUMN_NAME)
.isEqualTo(incidentStatusChangeParameters.getIncidentId());
if (existingRowsIncidentIdSelection.isEmpty()) {
// incident not found, skipping because changing the status is a background async operation that is not returning the result
Expand All @@ -688,7 +688,7 @@ public IncidentNotificationMessage changeIncidentStatus(IncidentStatusChangePara
this.newIncidentByHashRowIndexes.put(incidentHash, new IntArrayList(new int[] { targetNewIncidentsRowIndex }));
}

TextColumn newRowStatusColumn = this.allNewIncidentRows.textColumn(IncidentsColumnNames.STATUS_COLUMN_NAME);
StringColumn newRowStatusColumn = this.allNewIncidentRows.stringColumn(IncidentsColumnNames.STATUS_COLUMN_NAME);
String currentStatus = newRowStatusColumn.get(targetNewIncidentsRowIndex);
if (!Objects.equals(currentStatus, newStatusString)) {
newRowStatusColumn.set(targetNewIncidentsRowIndex, newStatusString);
Expand Down Expand Up @@ -727,15 +727,15 @@ public void changeIncidentIssueUrl(IncidentIssueUrlChangeParameters incidentIssu
this.existingIncidentByHashRowIndexes = findIncidentRowIndexes(this.allExistingIncidentRows);
}

Selection newRowsIncidentIdSelection = this.allNewIncidentRows.textColumn(IncidentsColumnNames.ID_COLUMN_NAME)
Selection newRowsIncidentIdSelection = this.allNewIncidentRows.stringColumn(IncidentsColumnNames.ID_COLUMN_NAME)
.isEqualTo(incidentIssueUrlChangeParameters.getIncidentId());
if (!newRowsIncidentIdSelection.isEmpty()) {
// the updated row has other updates awaiting
int newRowIndex = newRowsIncidentIdSelection.get(0);
TextColumn newRowIssueUrlColumn = this.allNewIncidentRows.textColumn(IncidentsColumnNames.ISSUE_URL_COLUMN_NAME);
StringColumn newRowIssueUrlColumn = this.allNewIncidentRows.stringColumn(IncidentsColumnNames.ISSUE_URL_COLUMN_NAME);
newRowIssueUrlColumn.set(newRowIndex, incidentIssueUrlChangeParameters.getNewIssueUrl());
} else if (this.allExistingIncidentRows != null) {
Selection existingRowsIncidentIdSelection = this.allExistingIncidentRows.textColumn(IncidentsColumnNames.ID_COLUMN_NAME)
Selection existingRowsIncidentIdSelection = this.allExistingIncidentRows.stringColumn(IncidentsColumnNames.ID_COLUMN_NAME)
.isEqualTo(incidentIssueUrlChangeParameters.getIncidentId());
if (existingRowsIncidentIdSelection.isEmpty()) {
// incident not found, skipping because changing the status is a background async operation that is not returning the result
Expand All @@ -757,7 +757,7 @@ public void changeIncidentIssueUrl(IncidentIssueUrlChangeParameters incidentIssu
this.newIncidentByHashRowIndexes.put(incidentHash, new IntArrayList(new int[] { targetNewIncidentsRowIndex }));
}

TextColumn newRowIssueUrlColumn = this.allNewIncidentRows.textColumn(IncidentsColumnNames.ISSUE_URL_COLUMN_NAME);
StringColumn newRowIssueUrlColumn = this.allNewIncidentRows.stringColumn(IncidentsColumnNames.ISSUE_URL_COLUMN_NAME);
newRowIssueUrlColumn.set(targetNewIncidentsRowIndex, incidentIssueUrlChangeParameters.getNewIssueUrl());
}
}
Expand Down
4 changes: 2 additions & 2 deletions dqops/src/main/java/com/dqops/core/jobqueue/jobs/data/RepairStoredDataQueueJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import tech.tablesaw.api.TextColumn;
import tech.tablesaw.api.StringColumn;

import java.util.List;

Expand Down Expand Up @@ -152,7 +152,7 @@ protected void loadAndFixMonthlyPartitions(FileStorageSettings fileStorageSettin
partitionId, fileStorageSettings, null, userIdentity);

if (loadedMonthlyPartition.getData() != null && loadedMonthlyPartition.getData().rowCount() != 0) {
TextColumn idColumn = loadedMonthlyPartition.getData().textColumn(fileStorageSettings.getIdStringColumnName());
StringColumn idColumn = loadedMonthlyPartition.getData().stringColumn(fileStorageSettings.getIdStringColumnName());
if (idColumn.countUnique() != idColumn.size()) {
// duplicates found, removing the partition
this.parquetPartitionStorageService.deletePartitionFile(loadedMonthlyPartition.getPartitionId(), fileStorageSettings, userIdentity);
Expand Down
9 changes: 3 additions & 6 deletions dqops/src/main/java/com/dqops/core/jobqueue/jobs/schema/ImportSchemaQueueJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,7 @@
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import tech.tablesaw.api.IntColumn;
import tech.tablesaw.api.Row;
import tech.tablesaw.api.Table;
import tech.tablesaw.api.TextColumn;
import tech.tablesaw.api.*;

import java.util.ArrayList;
import java.util.List;
Expand Down Expand Up @@ -127,8 +124,8 @@ public List<TableSpec> filterTableSpecs(List<TableSpec> baseSpecs, String tableF
*/
public Table createDatasetTableFromTableSpecs(List<TableSpec> sourceTableSpecs) {
Table resultTable = Table.create().addColumns(
TextColumn.create("Schema name"),
TextColumn.create("Table name"),
StringColumn.create("Schema name"),
StringColumn.create("Table name"),
IntColumn.create("Column count"));

for( TableSpec sourceTableSpec : sourceTableSpecs) {
Expand Down
9 changes: 3 additions & 6 deletions dqops/src/main/java/com/dqops/core/jobqueue/jobs/table/ImportTablesQueueJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,7 @@
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import tech.tablesaw.api.IntColumn;
import tech.tablesaw.api.Row;
import tech.tablesaw.api.Table;
import tech.tablesaw.api.TextColumn;
import tech.tablesaw.api.*;

import java.util.ArrayList;
import java.util.List;
Expand Down Expand Up @@ -111,8 +108,8 @@ public Table createDatasetTableFromTableSpecs(List<TableSpec> sourceTableSpecs)
*/
private Table createEmptyOutputResult() {
return Table.create().addColumns(
TextColumn.create("Schema name"),
TextColumn.create("Table name"),
StringColumn.create("Schema name"),
StringColumn.create("Table name"),
IntColumn.create("Column count"));
}

Expand Down
8 changes: 4 additions & 4 deletions dqops/src/main/java/com/dqops/data/checkresults/factory/CheckResultsTableFactoryImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,10 @@ public Table createEmptyCheckResultsTable(String tableName) {
table.addColumns(
IntColumn.create(CheckResultsColumnNames.SEVERITY_COLUMN_NAME),
LongColumn.create(CheckResultsColumnNames.INCIDENT_HASH_COLUMN_NAME),
TextColumn.create(CheckResultsColumnNames.REFERENCE_CONNECTION_COLUMN_NAME),
TextColumn.create(CheckResultsColumnNames.REFERENCE_SCHEMA_COLUMN_NAME),
TextColumn.create(CheckResultsColumnNames.REFERENCE_TABLE_COLUMN_NAME),
TextColumn.create(CheckResultsColumnNames.REFERENCE_COLUMN_COLUMN_NAME),
StringColumn.create(CheckResultsColumnNames.REFERENCE_CONNECTION_COLUMN_NAME),
StringColumn.create(CheckResultsColumnNames.REFERENCE_SCHEMA_COLUMN_NAME),
StringColumn.create(CheckResultsColumnNames.REFERENCE_TABLE_COLUMN_NAME),
StringColumn.create(CheckResultsColumnNames.REFERENCE_COLUMN_COLUMN_NAME),
BooleanColumn.create(CheckResultsColumnNames.INCLUDE_IN_KPI_COLUMN_NAME),
BooleanColumn.create(CheckResultsColumnNames.INCLUDE_IN_SLA_COLUMN_NAME),
DoubleColumn.create(CheckResultsColumnNames.FATAL_LOWER_BOUND_COLUMN_NAME),
Expand Down
Loading

0 comments on commit a389811

Please sign in to comment.