Skip to content
This repository has been archived by the owner on Jun 12, 2024. It is now read-only.

Commit

Permalink
fix import bug and add ref support (#88)
Browse files Browse the repository at this point in the history 
* fix import bug and add ref support

* fix calls

* add docs
  • Loading branch information
@hay-kot
hay-kot authored Oct 16, 2022
1 parent 5596740 commit dbaaf4a
Show file tree
Hide file tree
Showing 8 changed files with 118 additions and 57 deletions.
2 changes: 1 addition & 1 deletion backend/app/api/demo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func (a *app) SetupDemo() {
log.Fatal().Msg("Failed to setup demo")
}

err = a.services.Items.CsvImport(context.Background(), self.GroupID, records)
_, err = a.services.Items.CsvImport(context.Background(), self.GroupID, records)
if err != nil {
log.Err(err).Msg("Failed to import CSV")
log.Fatal().Msg("Failed to setup demo")
Expand Down
2 changes: 1 addition & 1 deletion backend/app/api/v1/v1_ctrl_items.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ func (ctrl *V1Controller) HandleItemsImport() http.HandlerFunc {

user := services.UseUserCtx(r.Context())

err = ctrl.svc.Items.CsvImport(r.Context(), user.GroupID, data)
_, err = ctrl.svc.Items.CsvImport(r.Context(), user.GroupID, data)
if err != nil {
log.Err(err).Msg("failed to import items")
server.RespondServerError(w)
Expand Down
6 changes: 6 additions & 0 deletions backend/internal/repo/repo_items.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,11 @@ func (e *ItemsRepository) GetOne(ctx context.Context, id uuid.UUID) (ItemOut, er
return e.getOne(ctx, item.ID(id))
}

func (e *ItemsRepository) CheckRef(ctx context.Context, GID uuid.UUID, ref string) (bool, error) {
q := e.db.Item.Query().Where(item.HasGroupWith(group.ID(GID)))
return q.Where(item.ImportRef(ref)).Exist(ctx)
}

// GetOneByGroup returns a single item by ID. If the item does not exist, an error is returned.
// GetOneByGroup ensures that the item belongs to a specific group.
func (e *ItemsRepository) GetOneByGroup(ctx context.Context, gid, id uuid.UUID) (ItemOut, error) {
Expand Down Expand Up @@ -287,6 +292,7 @@ func (e *ItemsRepository) GetAll(ctx context.Context, gid uuid.UUID) ([]ItemSumm

func (e *ItemsRepository) Create(ctx context.Context, gid uuid.UUID, data ItemCreate) (ItemOut, error) {
q := e.db.Item.Create().
SetImportRef(data.ImportRef).
SetName(data.Name).
SetDescription(data.Description).
SetGroupID(gid).
Expand Down
80 changes: 55 additions & 25 deletions backend/internal/services/service_items.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package services
import (
"context"
"errors"
"fmt"

"github.com/google/uuid"
"github.com/hay-kot/homebox/backend/internal/repo"
Expand Down Expand Up @@ -48,7 +47,7 @@ func (svc *ItemService) Update(ctx context.Context, gid uuid.UUID, data repo.Ite
return svc.repo.Items.UpdateByGroup(ctx, gid, data)
}

func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]string) error {
func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]string) (int, error) {
loaded := []csvRow{}

// Skip first row
Expand All @@ -59,18 +58,41 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
}

if len(row) != NumOfCols {
return ErrInvalidCsv
return 0, ErrInvalidCsv
}

r := newCsvRow(row)
loaded = append(loaded, r)
}

// validate rows
var errMap = map[int][]error{}
var hasErr bool
for i, r := range loaded {

errs := r.validate()

if len(errs) > 0 {
hasErr = true
lineNum := i + 2

errMap[lineNum] = errs
}
}

if hasErr {
for lineNum, errs := range errMap {
for _, err := range errs {
log.Error().Err(err).Int("line", lineNum).Msg("csv import error")
}
}
}

// Bootstrap the locations and labels so we can reuse the created IDs for the items
locations := map[string]uuid.UUID{}
existingLocation, err := svc.repo.Locations.GetAll(ctx, gid)
if err != nil {
return err
return 0, err
}
for _, loc := range existingLocation {
locations[loc.Name] = loc.ID
Expand All @@ -79,7 +101,7 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
labels := map[string]uuid.UUID{}
existingLabels, err := svc.repo.Labels.GetAll(ctx, gid)
if err != nil {
return err
return 0, err
}
for _, label := range existingLabels {
labels[label.Name] = label.ID
Expand All @@ -88,40 +110,48 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
for _, row := range loaded {

// Locations
if _, ok := locations[row.Location]; ok {
continue
}

fmt.Println("Creating Location: ", row.Location)

result, err := svc.repo.Locations.Create(ctx, gid, repo.LocationCreate{
Name: row.Location,
Description: "",
})
if err != nil {
return err
if _, exists := locations[row.Location]; !exists {
result, err := svc.repo.Locations.Create(ctx, gid, repo.LocationCreate{
Name: row.Location,
Description: "",
})
if err != nil {
return 0, err
}
locations[row.Location] = result.ID
}
locations[row.Location] = result.ID

// Labels

for _, label := range row.getLabels() {
if _, ok := labels[label]; ok {
if _, exists := labels[label]; exists {
continue
}
result, err := svc.repo.Labels.Create(ctx, gid, repo.LabelCreate{
Name: label,
Description: "",
})
if err != nil {
return err
return 0, err
}
labels[label] = result.ID
}
}

// Create the items
var count int
for _, row := range loaded {
// Check Import Ref
if row.Item.ImportRef != "" {
exists, err := svc.repo.Items.CheckRef(ctx, gid, row.Item.ImportRef)
if exists {
continue
}
if err != nil {
log.Err(err).Msg("error checking import ref")
}
}

locationID := locations[row.Location]
labelIDs := []uuid.UUID{}
for _, label := range row.getLabels() {
Expand All @@ -131,8 +161,6 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
log.Info().
Str("name", row.Item.Name).
Str("location", row.Location).
Strs("labels", row.getLabels()).
Str("locationId", locationID.String()).
Msgf("Creating Item: %s", row.Item.Name)

result, err := svc.repo.Items.Create(ctx, gid, repo.ItemCreate{
Expand All @@ -144,7 +172,7 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
})

if err != nil {
return err
return count, err
}

// Update the item with the rest of the data
Expand Down Expand Up @@ -183,8 +211,10 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s
})

if err != nil {
return err
return count, err
}

count++
}
return nil
return count, nil
}
19 changes: 19 additions & 0 deletions backend/internal/services/service_items_csv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,3 +97,22 @@ func (c csvRow) getLabels() []string {

return split
}

func (c csvRow) validate() []error {
var errs []error

add := func(err error) {
errs = append(errs, err)
}

required := func(s string, name string) {
if s == "" {
add(errors.New(name + " is required"))
}
}

required(c.Location, "Location")
required(c.Item.Name, "Name")

return errs
}
12 changes: 6 additions & 6 deletions backend/internal/services/service_items_csv_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ import (

const CSV_DATA = `
Import Ref,Location,Labels,Quantity,Name,Description,Insured,Serial Number,Mode Number,Manufacturer,Notes,Purchase From,Purchased Price,Purchased Time,Lifetime Warranty,Warranty Expires,Warranty Details,Sold To,Sold Price,Sold Time,Sold Notes
,Garage,IOT;Home Assistant; Z-Wave,1,Zooz Universal Relay ZEN17,"Zooz 700 Series Z-Wave Universal Relay ZEN17 for Awnings, Garage Doors, Sprinklers, and More | 2 NO-C-NC Relays (20A, 10A) | Signal Repeater | Hub Required (Compatible with SmartThings and Hubitat)",,,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,,,,,,
,Living Room,IOT;Home Assistant; Z-Wave,1,Zooz Motion Sensor,"Zooz Z-Wave Plus S2 Motion Sensor ZSE18 with Magnetic Mount, Works with Vera and SmartThings",,,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,,,,,,
,Office,IOT;Home Assistant; Z-Wave,1,Zooz 110v Power Switch,"Zooz Z-Wave Plus Power Switch ZEN15 for 110V AC Units, Sump Pumps, Humidifiers, and More",,,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,,,,,,
,Downstairs,IOT;Home Assistant; Z-Wave,1,Ecolink Z-Wave PIR Motion Sensor,"Ecolink Z-Wave PIR Motion Detector Pet Immune, White (PIRZWAVE2.5-ECO)",,,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,,,,,,
,Entry,IOT;Home Assistant; Z-Wave,1,Yale Security Touchscreen Deadbolt,"Yale Security YRD226-ZW2-619 YRD226ZW2619 Touchscreen Deadbolt, Satin Nickel",,,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,,,,,,
,Kitchen,IOT;Home Assistant; Z-Wave,1,Smart Rocker Light Dimmer,"UltraPro Z-Wave Smart Rocker Light Dimmer with QuickFit and SimpleWire, 3-Way Ready, Compatible with Alexa, Google Assistant, ZWave Hub Required, Repeater/Range Extender, White Paddle Only, 39351",,,39351,Honeywell,,Amazon,65.98,09/30/0202,,,,,,,`
A,Garage,IOT;Home Assistant; Z-Wave,1,Zooz Universal Relay ZEN17,"Zooz 700 Series Z-Wave Universal Relay ZEN17 for Awnings, Garage Doors, Sprinklers, and More | 2 NO-C-NC Relays (20A, 10A) | Signal Repeater | Hub Required (Compatible with SmartThings and Hubitat)",,,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,,,,,,
B,Living Room,IOT;Home Assistant; Z-Wave,1,Zooz Motion Sensor,"Zooz Z-Wave Plus S2 Motion Sensor ZSE18 with Magnetic Mount, Works with Vera and SmartThings",,,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,,,,,,
C,Office,IOT;Home Assistant; Z-Wave,1,Zooz 110v Power Switch,"Zooz Z-Wave Plus Power Switch ZEN15 for 110V AC Units, Sump Pumps, Humidifiers, and More",,,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,,,,,,
D,Downstairs,IOT;Home Assistant; Z-Wave,1,Ecolink Z-Wave PIR Motion Sensor,"Ecolink Z-Wave PIR Motion Detector Pet Immune, White (PIRZWAVE2.5-ECO)",,,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,,,,,,
E,Entry,IOT;Home Assistant; Z-Wave,1,Yale Security Touchscreen Deadbolt,"Yale Security YRD226-ZW2-619 YRD226ZW2619 Touchscreen Deadbolt, Satin Nickel",,,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,,,,,,
F,Kitchen,IOT;Home Assistant; Z-Wave,1,Smart Rocker Light Dimmer,"UltraPro Z-Wave Smart Rocker Light Dimmer with QuickFit and SimpleWire, 3-Way Ready, Compatible with Alexa, Google Assistant, ZWave Hub Required, Repeater/Range Extender, White Paddle Only, 39351",,,39351,Honeywell,,Amazon,65.98,09/30/0202,,,,,,,`

func loadcsv() [][]string {
reader := csv.NewReader(bytes.NewBuffer([]byte(CSV_DATA)))
Expand Down
8 changes: 7 additions & 1 deletion backend/internal/services/service_items_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,13 @@ func TestItemService_CsvImport(t *testing.T) {
svc := &ItemService{
repo: tRepos,
}
err := svc.CsvImport(context.Background(), tGroup.ID, data)
count, err := svc.CsvImport(context.Background(), tGroup.ID, data)
assert.Equal(t, 6, count)
assert.NoError(t, err)

// Check import refs are deduplicated
count, err = svc.CsvImport(context.Background(), tGroup.ID, data)
assert.Equal(t, 0, count)
assert.NoError(t, err)

items, err := svc.GetAll(context.Background(), tGroup.ID)
Expand Down
46 changes: 23 additions & 23 deletions docs/docs/import-csv.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,29 +23,29 @@ Import RefLocation Labels Quantity Name Description Insured Serial Number Model

## CSV Reference

| Column | Type | Description |
| ----------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------- |
| ImportRef | String (100) | Future |
| Location | String | This is the location of the item that will be created. These are de-duplicated and won't create another instance when reused. |
| Labels | `;` Separated String | List of labels to apply to the item separated by a `;`, can be existing or new |
| Quantity | Integer | The quantity of items to create |
| Name | String | Name of the item |
| Description | String | Description of the item |
| Insured | Boolean | Whether or not the item is insured |
| Serial Number | String | Serial number of the item |
| Model Number | String | Model of the item |
| Manufacturer | String | Manufacturer of the item |
| Notes | String (1000) | General notes about the product |
| Purchase From | String | Name of the place the item was purchased from |
| Purchase Price | Float64 | |
| Purchase At | Date | Date the item was purchased |
| Lifetime Warranty | Boolean | true or false - case insensitive |
| Warranty Expires | Date | Date in the format |
| Warranty Details | String | Details about the warranty |
| Sold To | String | Name of the person the item was sold to |
| Sold At | Date | Date the item was sold |
| Sold Price | Float64 | |
| Sold Notes | String (1000) | |
| Column | Type | Description |
| ----------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| ImportRef | String (100) | Import Refs are unique strings that can be used to deduplicate imports. Before an item is imported, we check the database for a matching ref. If the ref exists, we skip that item. |
| Location | String | This is the location of the item that will be created. These are de-duplicated and won't create another instance when reused. |
| Labels | `;` Separated String | List of labels to apply to the item separated by a `;`, can be existing or new |
| Quantity | Integer | The quantity of items to create |
| Name | String | Name of the item |
| Description | String | Description of the item |
| Insured | Boolean | Whether or not the item is insured |
| Serial Number | String | Serial number of the item |
| Model Number | String | Model of the item |
| Manufacturer | String | Manufacturer of the item |
| Notes | String (1000) | General notes about the product |
| Purchase From | String | Name of the place the item was purchased from |
| Purchase Price | Float64 | |
| Purchase At | Date | Date the item was purchased |
| Lifetime Warranty | Boolean | true or false - case insensitive |
| Warranty Expires | Date | Date in the format |
| Warranty Details | String | Details about the warranty |
| Sold To | String | Name of the person the item was sold to |
| Sold At | Date | Date the item was sold |
| Sold Price | Float64 | |
| Sold Notes | String (1000) | |

**Type Key**

Expand Down

0 comments on commit dbaaf4a

Please sign in to comment.