Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: autodetect data type during import #854

Merged
merged 1 commit into from
Feb 28, 2024
Merged

feat: autodetect data type during import #854

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions lib/Service/ColumnService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,7 @@ public function delete(int $id, bool $skipRowCleanup = false, ?string $userId =
* @param int|null $tableId
* @param int|null $viewId
* @param array $titles example ['Test column 1', 'And so on', '3rd column title']
* @param array $dataTypes example ['datetime', 'number', 'text']
* @param string|null $userId
* @param bool $createUnknownColumns
* @param int $countCreatedColumns
Expand All @@ -488,7 +489,7 @@ public function delete(int $id, bool $skipRowCleanup = false, ?string $userId =
* @throws NotFoundError
* @throws PermissionError
*/
public function findOrCreateColumnsByTitleForTableAsArray(?int $tableId, ?int $viewId, array $titles, ?string $userId, bool $createUnknownColumns, int &$countCreatedColumns, int &$countMatchingColumns): array {
public function findOrCreateColumnsByTitleForTableAsArray(?int $tableId, ?int $viewId, array $titles, array $dataTypes, ?string $userId, bool $createUnknownColumns, int &$countCreatedColumns, int &$countMatchingColumns): array {
$result = [];

if($userId === null) {
Expand Down Expand Up @@ -522,7 +523,29 @@ public function findOrCreateColumnsByTitleForTableAsArray(?int $tableId, ?int $v
// if column was not found
if($result[$i] === '' && $createUnknownColumns) {
$description = $this->l->t('This column was automatically created by the import service.');
$result[$i] = $this->create($userId, $tableId, $viewId, 'text', 'line', $title, false, $description, null, null, null, null, null, null, null, null, null, null, null, null, []);
$result[$i] = $this->create(
$userId,
$tableId,
$viewId,
$dataTypes[$i]['type'],
$dataTypes[$i]['subtype'] ?? '',
$title,
false,
$description,
null,
null,
null,
$dataTypes[$i]['number_prefix'] ?? null,
$dataTypes[$i]['number_suffix'] ?? null,
null,
null,
null,
$dataTypes[$i]['number_decimals'] ?? null,
null,
$dataTypes[$i]['selection_default'] ?? null,
null,
[]
);
$countCreatedColumns++;
}
}
Expand Down
122 changes: 104 additions & 18 deletions lib/Service/ImportService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@
use OCP\Files\NotPermittedException;
use OCP\IUserManager;
use OCP\Server;
use PhpOffice\PhpSpreadsheet\Cell\Cell;
use PhpOffice\PhpSpreadsheet\Cell\DataType;
use PhpOffice\PhpSpreadsheet\IOFactory;
use PhpOffice\PhpSpreadsheet\Shared\Date;
use PhpOffice\PhpSpreadsheet\Worksheet\Row;
use PhpOffice\PhpSpreadsheet\Worksheet\Worksheet;
use Psr\Container\ContainerExceptionInterface;
Expand Down Expand Up @@ -145,18 +148,17 @@ public function import(?int $tableId, ?int $viewId, string $path, bool $createMi
* @throws PermissionError
*/
private function loop(Worksheet $worksheet): void {
$firstRow = true;
foreach ($worksheet->getRowIterator() as $row) {
if ($firstRow) {
$this->getColumns($row);
if (empty(array_filter($this->columns))) {
return;
}
$firstRow = false;
} else {
// parse row data
$this->createRow($row);
}
$firstRow = $worksheet->getRowIterator()->current();
$secondRow = $worksheet->getRowIterator()->seek(2)->current();
$this->getColumns($firstRow, $secondRow);

if (empty(array_filter($this->columns))) {
return;
}

foreach ($worksheet->getRowIterator(2) as $row) {
// parse row data
$this->createRow($row);
}
}

Expand Down Expand Up @@ -204,20 +206,32 @@ private function createRow(Row $row): void {
continue;
}

/** @var Column $column */
$column = $this->columns[$i];

// if cell is empty
if(!$cell || $cell->getValue() === null) {
$this->logger->info('Cell is empty while fetching rows data for importing.');
if($this->columns[$i]->getMandatory()) {
if($column->getMandatory()) {
$this->logger->warning('Mandatory column was not set');
$this->countErrors++;
return;
}
continue;
}

$value = $cell->getValue();
if ($column->getType() === 'datetime') {
$value = Date::excelToDateTimeObject($value)->format('Y-m-d H:i');
} elseif ($column->getType() === 'number' && $column->getNumberSuffix() === '%') {
$value = $value * 100;
} elseif ($column->getType() === 'selection' && $column->getSubtype() === 'check') {
$value = $cell->getFormattedValue() === 'TRUE' ? 'true' : 'false';
}

$data[] = [
'columnId' => (int) $this->columns[$i]->getId(),
'value' => json_decode($this->parseValueByColumnType($cell->getValue(), $this->columns[$i])),
'value' => json_decode($this->parseValueByColumnType($value, $this->columns[$i])),
];
}
try {
Expand All @@ -237,29 +251,101 @@ private function createRow(Row $row): void {
}

/**
* @param Row $row
* @param Row $firstRow
* @param Row $secondRow
* @throws InternalError
* @throws NotFoundError
* @throws PermissionError
* @throws DoesNotExistException
* @throws MultipleObjectsReturnedException
*/
private function getColumns(Row $row): void {
$cellIterator = $row->getCellIterator();
private function getColumns(Row $firstRow, Row $secondRow): void {
$cellIterator = $firstRow->getCellIterator();
$secondRowCellIterator = $secondRow->getCellIterator();
$titles = [];
$dataTypes = [];
foreach ($cellIterator as $cell) {
if ($cell && $cell->getValue() !== null && $cell->getValue() !== '') {
$titles[] = $cell->getValue();

// Convert data type to our data type
$dataTypes[] = $this->parseColumnDataType($secondRowCellIterator->current());
} else {
$this->logger->debug('No cell given or cellValue is empty while loading columns for importing');
$this->countErrors++;
}
$secondRowCellIterator->next();
}
try {
$this->columns = $this->columnService->findOrCreateColumnsByTitleForTableAsArray($this->tableId, $this->viewId, $titles, $this->userId, $this->createUnknownColumns, $this->countCreatedColumns, $this->countMatchingColumns);
$this->columns = $this->columnService->findOrCreateColumnsByTitleForTableAsArray($this->tableId, $this->viewId, $titles, $dataTypes, $this->userId, $this->createUnknownColumns, $this->countCreatedColumns, $this->countMatchingColumns);
} catch (Exception $e) {
throw new InternalError($e->getMessage());
}
}

private function parseColumnDataType(Cell $cell): array {
$originDataType = $cell->getDataType();
$value = $cell->getValue();
$formattedValue = $cell->getFormattedValue();
$dataType = [
'type' => 'text',
'subtype' => 'line',
];

if (Date::isDateTime($cell) || $originDataType === DataType::TYPE_ISO_DATE) {
$dataType = [
'type' => 'datetime',
];
} elseif ($originDataType === DataType::TYPE_NUMERIC) {
if (str_contains($formattedValue, '%')) {
$dataType = [
'type' => 'number',
'number_decimals' => 2,
'number_suffix' => '%',
];
} elseif (str_contains($formattedValue, '€')) {
$dataType = [
'type' => 'number',
'number_decimals' => 2,
'number_suffix' => '€',
];
} elseif (str_contains($formattedValue, 'EUR')) {
$dataType = [
'type' => 'number',
'number_decimals' => 2,
'number_suffix' => 'EUR',
];
} elseif (str_contains($formattedValue, '$')) {
$dataType = [
'type' => 'number',
'number_decimals' => 2,
'number_prefix' => '$',
];
} elseif (str_contains($formattedValue, 'USD')) {
$dataType = [
'type' => 'number',
'number_decimals' => 2,
'number_suffix' => 'USD',
];
} elseif (is_float($value)) {
$decimals = strlen(substr(strrchr((string)$value, "."), 1));
$dataType = [
'type' => 'number',
'number_decimals' => $decimals,
];
} else {
$dataType = [
'type' => 'number',
];
}
} elseif ($originDataType === DataType::TYPE_BOOL) {
$dataType = [
'type' => 'selection',
'subtype' => 'check',
'selection_default' => 'false',
];
}

return $dataType;
}
}
Loading