Merge pull request #395 from getAlby/fix/add-fee-provision-2

Fix/add fee provision 2

db/migrations/20230703120000_add_tx_entry_type.up.sql

@@ -0,0 +1,2 @@
+alter table transaction_entries 
+add column entry_type character varying;

db/migrations/20230703130000_replace_tx_entry_constraint.up.sql

@@ -0,0 +1,2 @@
+alter table transaction_entries drop constraint unique_tx_entry_tuple,
+add constraint unique_tx_entry_tuple UNIQUE(user_id, invoice_id, debit_account_id, credit_account_id, entry_type);

db/models/transactionentry.go

@@ -4,6 +4,15 @@ import (
  "time"
 )
 
+const (
+ EntryTypeIncoming = "incoming"
+ EntryTypeOutgoing = "outgoing"
+ EntryTypeFee = "fee"
+ EntryTypeFeeReserve = "fee_reserve"
+ EntryTypeFeeReserveReversal = "fee_reserve_reversal"
+ EntryTypeOutgoingReversal = "outgoing_reversal"
+)
+
 // TransactionEntry : Transaction Entries Model
 type TransactionEntry struct {
  ID int64 `bun:",pk,autoincrement"`
@@ -14,9 +23,11 @@ type TransactionEntry struct {
  ParentID int64 `bun:",nullzero"`
  Parent *TransactionEntry `bun:"rel:belongs-to"`
  CreditAccountID int64 `bun:",notnull"`
+ FeeReserve *TransactionEntry `bun:"rel:belongs-to"`
  CreditAccount *Account `bun:"rel:belongs-to,join:credit_account_id=id"`
  DebitAccountID int64 `bun:",notnull"`
  DebitAccount *Account `bun:"rel:belongs-to,join:debit_account_id=id"`
  Amount int64 `bun:",notnull"`
  CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"`
+ EntryType string
 }

integration_tests/hodl_invoice_test.go

@@ -28,13 +28,14 @@ type HodlInvoiceSuite struct {
  service *service.LndhubService
  userLogin ExpectedCreateUserResponseBody
  userToken string
+ userToken2 string
  invoiceUpdateSubCancelFn context.CancelFunc
  hodlLND *LNDMockHodlWrapperAsync
 }
 
 func (suite *HodlInvoiceSuite) SetupSuite() {
  mlnd := newDefaultMockLND()
- externalLND, err := NewMockLND("1234567890abcdefabcd", 0, make(chan (*lnrpc.Invoice)))
+ externalLND, err := NewMockLND("1234567890abcdefabcd", 0, make(chan (*lnrpc.Invoice), 5))
  if err != nil {
  log.Fatalf("Error initializing test service: %v", err)
  }
@@ -51,7 +52,7 @@ func (suite *HodlInvoiceSuite) SetupSuite() {
  if err != nil {
  log.Fatalf("Error initializing test service: %v", err)
  }
- users, userTokens, err := createUsers(svc, 1)
+ users, userTokens, err := createUsers(svc, 2)
  if err != nil {
  log.Fatalf("Error creating test users: %v", err)
  }
@@ -66,21 +67,22 @@ func (suite *HodlInvoiceSuite) SetupSuite() {
  e.HTTPErrorHandler = responses.HTTPErrorHandler
  e.Validator = &lib.CustomValidator{Validator: validator.New()}
  suite.echo = e
- assert.Equal(suite.T(), 1, len(users))
- assert.Equal(suite.T(), 1, len(userTokens))
+ assert.Equal(suite.T(), 2, len(users))
+ assert.Equal(suite.T(), 2, len(userTokens))
  suite.userLogin = users[0]
  suite.userToken = userTokens[0]
+ suite.userToken2 = userTokens[1]
  suite.echo.Use(tokens.Middleware([]byte(suite.service.Config.JWTSecret)))
  suite.echo.GET("/balance", controllers.NewBalanceController(suite.service).Balance)
  suite.echo.POST("/addinvoice", controllers.NewAddInvoiceController(suite.service).AddInvoice)
  suite.echo.POST("/payinvoice", controllers.NewPayInvoiceController(suite.service).PayInvoice)
 }
 
 func (suite *HodlInvoiceSuite) TestHodlInvoice() {
- userFundingSats := 1000
- externalSatRequested := 500
+ userFundingSats := int64(1000)
+ externalSatRequested := int64(500)
  // fund user account
- invoiceResponse := suite.createAddInvoiceReq(userFundingSats, "integration test external payment user", suite.userToken)
+ invoiceResponse := suite.createAddInvoiceReq(int(userFundingSats), "integration test external payment user", suite.userToken)
  err := suite.mlnd.mockPaidInvoice(invoiceResponse, 0, false, nil)
  assert.NoError(suite.T(), err)
 
@@ -90,7 +92,7 @@ func (suite *HodlInvoiceSuite) TestHodlInvoice() {
  // create external invoice
  externalInvoice := lnrpc.Invoice{
  Memo: "integration tests: external pay from user",
- Value: int64(externalSatRequested),
+ Value: externalSatRequested,
  RPreimage: []byte("preimage1"),
  }
  invoice, err := suite.externalLND.AddInvoice(context.Background(), &externalInvoice)
@@ -107,7 +109,10 @@ func (suite *HodlInvoiceSuite) TestHodlInvoice() {
  if err != nil {
  fmt.Printf("Error when getting balance %v\n", err.Error())
  }
- assert.Equal(suite.T(), int64(userFundingSats-externalSatRequested), userBalance)
+
+ //also check that the fee reserve was reduced
+ feeReserve := suite.service.CalcFeeLimit(suite.externalLND.GetMainPubkey(), int64(externalSatRequested))
+ assert.Equal(suite.T(), userFundingSats-externalSatRequested-feeReserve, userBalance)
 
  // check payment is pending
  inv, err := suite.service.FindInvoiceByPaymentHash(context.Background(), userId, hex.EncodeToString(invoice.RHash))
@@ -153,16 +158,23 @@ func (suite *HodlInvoiceSuite) TestHodlInvoice() {
  errorString := "FAILURE_REASON_INCORRECT_PAYMENT_DETAILS"
  assert.Equal(suite.T(), errorString, invoices[0].ErrorMessage)
 
- transactonEntries, err := suite.service.TransactionEntriesFor(context.Background(), userId)
+ transactionEntries, err := suite.service.TransactionEntriesFor(context.Background(), userId)
  if err != nil {
  fmt.Printf("Error when getting transaction entries %v\n", err.Error())
  }
- // check if there are 3 transaction entries, with reversed credit and debit account ids
- assert.Equal(suite.T(), 3, len(transactonEntries))
- assert.Equal(suite.T(), transactonEntries[1].CreditAccountID, transactonEntries[2].DebitAccountID)
- assert.Equal(suite.T(), transactonEntries[1].DebitAccountID, transactonEntries[2].CreditAccountID)
- assert.Equal(suite.T(), transactonEntries[1].Amount, int64(externalSatRequested))
- assert.Equal(suite.T(), transactonEntries[2].Amount, int64(externalSatRequested))
+ // check if there are 5 transaction entries:
+ // - the incoming payment
+ // - the outgoing payment
+ // - the fee reserve + the fee reserve reversal
+ // - the outgoing payment reversal
+ // with reversed credit and debit account ids for payment 2/5 & payment 3/4
+ assert.Equal(suite.T(), 5, len(transactionEntries))
+ assert.Equal(suite.T(), transactionEntries[1].CreditAccountID, transactionEntries[4].DebitAccountID)
+ assert.Equal(suite.T(), transactionEntries[1].DebitAccountID, transactionEntries[4].CreditAccountID)
+ assert.Equal(suite.T(), transactionEntries[2].CreditAccountID, transactionEntries[3].DebitAccountID)
+ assert.Equal(suite.T(), transactionEntries[2].DebitAccountID, transactionEntries[3].CreditAccountID)
+ assert.Equal(suite.T(), transactionEntries[1].Amount, int64(externalSatRequested))
+ assert.Equal(suite.T(), transactionEntries[4].Amount, int64(externalSatRequested))
 
  // create external invoice
  externalInvoice = lnrpc.Invoice{
@@ -214,12 +226,97 @@ func (suite *HodlInvoiceSuite) TestHodlInvoice() {
  inv, err = suite.service.FindInvoiceByPaymentHash(context.Background(), userId, hex.EncodeToString(invoice.RHash))
  assert.NoError(suite.T(), err)
  assert.Equal(suite.T(), common.InvoiceStateSettled, inv.State)
- clearTable(suite.service, "invoices")
- clearTable(suite.service, "transaction_entries")
- clearTable(suite.service, "accounts")
+}
+func (suite *HodlInvoiceSuite) TestNegativeBalanceWithHodl() {
+ //10M funding, 5M sat requested
+ userFundingSats := 10000000
+ externalSatRequested := 5000000
+ userId := getUserIdFromToken(suite.userToken2)
+ // fund user account
+ invoiceResponse := suite.createAddInvoiceReq(userFundingSats, "integration test external payment user", suite.userToken2)
+ err := suite.mlnd.mockPaidInvoice(invoiceResponse, 0, false, nil)
+ assert.NoError(suite.T(), err)
+
+ // wait a bit for the callback event to hit
+ time.Sleep(10 * time.Millisecond)
+
+ // create external invoice
+ externalInvoice := lnrpc.Invoice{
+ Memo: "integration tests: external pay from user",
+ Value: int64(externalSatRequested),
+ RPreimage: []byte("preimage3"),
+ }
+ invoice, err := suite.externalLND.AddInvoice(context.Background(), &externalInvoice)
+ assert.NoError(suite.T(), err)
+ //the fee should be 1 %, so 50k sats (+1)
+ feeLimit := suite.service.CalcFeeLimit(suite.externalLND.GetMainPubkey(), int64(externalSatRequested))
+ // pay external from user, req will be canceled after 2 sec
+ go suite.createPayInvoiceReqWithCancel(invoice.PaymentRequest, suite.userToken2)
+ // wait for payment to be updated as pending in database
+ time.Sleep(3 * time.Second)
+ // check payment is pending
+ inv, err := suite.service.FindInvoiceByPaymentHash(context.Background(), userId, hex.EncodeToString(invoice.RHash))
+ assert.NoError(suite.T(), err)
+ assert.Equal(suite.T(), common.InvoiceStateInitialized, inv.State)
+
+ //drain balance from account: at this point we have 5 M, so we can pay 4.95M
+ drainInv := lnrpc.Invoice{
+ Memo: "integration tests: external pay from user",
+ Value: int64(4950000),
+ RPreimage: []byte("preimage4"),
+ }
+ drainInvoice, err := suite.externalLND.AddInvoice(context.Background(), &drainInv)
+ assert.NoError(suite.T(), err)
+ //pay drain invoice
+ go suite.createPayInvoiceReqWithCancel(drainInvoice.PaymentRequest, suite.userToken2)
+ time.Sleep(3 * time.Second)
+
+ //start payment checking loop
+ go func() {
+ err = suite.service.CheckAllPendingOutgoingPayments(context.Background())
+ assert.NoError(suite.T(), err)
+ }()
+ //wait a bit for routine to start
+ time.Sleep(time.Second)
+ //now settle both invoices with the maximum fee
+ suite.hodlLND.SettlePayment(lnrpc.Payment{
+ PaymentHash: hex.EncodeToString(drainInvoice.RHash),
+ Value: drainInv.Value,
+ CreationDate: 0,
+ FeeSat: feeLimit,
+ PaymentPreimage: "preimage3",
+ ValueSat: drainInv.Value,
+ ValueMsat: 0,
+ PaymentRequest: invoice.PaymentRequest,
+ Status: lnrpc.Payment_SUCCEEDED,
+ FailureReason: 0,
+ })
+ //wait a bit for db update to happen
+ time.Sleep(time.Second)
+ //send settle invoice with lnrpc.payment
+ suite.hodlLND.SettlePayment(lnrpc.Payment{
+ PaymentHash: hex.EncodeToString(invoice.RHash),
+ Value: externalInvoice.Value,
+ CreationDate: 0,
+ FeeSat: feeLimit,
+ PaymentPreimage: "preimage4",
+ ValueSat: externalInvoice.Value,
+ ValueMsat: 0,
+ PaymentRequest: invoice.PaymentRequest,
+ Status: lnrpc.Payment_SUCCEEDED,
+ FailureReason: 0,
+ })
+ //fetch user balance again
+ userBalance, err := suite.service.CurrentUserBalance(context.Background(), userId)
+ assert.NoError(suite.T(), err)
+ //assert the balance did not go below 0
+ assert.False(suite.T(), userBalance < 0)
 }
 
 func (suite *HodlInvoiceSuite) TearDownSuite() {
+ clearTable(suite.service, "invoices")
+ clearTable(suite.service, "transaction_entries")
+ clearTable(suite.service, "accounts")
  suite.invoiceUpdateSubCancelFn()
 }
 

integration_tests/internal_payment_test.go

@@ -171,13 +171,11 @@ func (suite *PaymentTestSuite) TestInternalPayment() {
  suite.echo.ServeHTTP(rec, req)
  assert.Equal(suite.T(), http.StatusBadRequest, rec.Code)
 
- transactonEntriesAlice, _ := suite.service.TransactionEntriesFor(context.Background(), aliceId)
+ transactionEntriesAlice, _ := suite.service.TransactionEntriesFor(context.Background(), aliceId)
  aliceBalance, _ := suite.service.CurrentUserBalance(context.Background(), aliceId)
- assert.Equal(suite.T(), 3, len(transactonEntriesAlice))
- assert.Equal(suite.T(), int64(aliceFundingSats), transactonEntriesAlice[0].Amount)
- assert.Equal(suite.T(), int64(bobSatRequested), transactonEntriesAlice[1].Amount)
- assert.Equal(suite.T(), int64(fee), transactonEntriesAlice[2].Amount)
- assert.Equal(suite.T(), transactonEntriesAlice[1].ID, transactonEntriesAlice[2].ParentID)
+ assert.Equal(suite.T(), 2, len(transactionEntriesAlice))
+ assert.Equal(suite.T(), int64(aliceFundingSats), transactionEntriesAlice[0].Amount)
+ assert.Equal(suite.T(), int64(bobSatRequested), transactionEntriesAlice[1].Amount)
  assert.Equal(suite.T(), int64(aliceFundingSats-bobSatRequested-fee), aliceBalance)
 
  bobBalance, _ := suite.service.CurrentUserBalance(context.Background(), bobId)
@@ -241,7 +239,7 @@ func (suite *PaymentTestSuite) TestInternalPaymentFail() {
  assert.Equal(suite.T(), 2, len(invoices))
  assert.Equal(suite.T(), common.InvoiceStateError, invoices[0].State)
  assert.Equal(suite.T(), common.InvoiceStateSettled, invoices[1].State)
- transactonEntries, err := suite.service.TransactionEntriesFor(context.Background(), userId)
+ transactionEntries, err := suite.service.TransactionEntriesFor(context.Background(), userId)
  if err != nil {
  fmt.Printf("Error when getting transaction entries %v\n", err.Error())
  }
@@ -251,15 +249,14 @@ func (suite *PaymentTestSuite) TestInternalPaymentFail() {
  fmt.Printf("Error when getting balance %v\n", err.Error())
  }
 
- // check if there are 5 transaction entries, with reversed credit and debit account ids for last 2
- assert.Equal(suite.T(), 5, len(transactonEntries))
- assert.Equal(suite.T(), int64(aliceFundingSats), transactonEntries[0].Amount)
- assert.Equal(suite.T(), int64(bobSatRequested), transactonEntries[1].Amount)
- assert.Equal(suite.T(), int64(fee), transactonEntries[2].Amount)
- assert.Equal(suite.T(), transactonEntries[3].CreditAccountID, transactonEntries[4].DebitAccountID)
- assert.Equal(suite.T(), transactonEntries[3].DebitAccountID, transactonEntries[4].CreditAccountID)
- assert.Equal(suite.T(), transactonEntries[3].Amount, int64(bobSatRequested))
- assert.Equal(suite.T(), transactonEntries[4].Amount, int64(bobSatRequested))
+ // check if there are 4 transaction entries, with reversed credit and debit account ids for last 2
+ assert.Equal(suite.T(), 4, len(transactionEntries))
+ assert.Equal(suite.T(), int64(aliceFundingSats), transactionEntries[0].Amount)
+ assert.Equal(suite.T(), int64(bobSatRequested), transactionEntries[1].Amount)
+ assert.Equal(suite.T(), transactionEntries[2].CreditAccountID, transactionEntries[3].DebitAccountID)
+ assert.Equal(suite.T(), transactionEntries[2].DebitAccountID, transactionEntries[3].CreditAccountID)
+ assert.Equal(suite.T(), transactionEntries[2].Amount, int64(bobSatRequested))
+ assert.Equal(suite.T(), transactionEntries[3].Amount, int64(bobSatRequested))
  // assert that balance was reduced only once
  assert.Equal(suite.T(), int64(aliceFundingSats)-int64(bobSatRequested+fee), int64(aliceBalance))
 }

integration_tests/lnd_mock_hodl.go

@@ -33,7 +33,7 @@ func (hps *HodlPaymentSubscriber) Recv() (*lnrpc.Payment, error) {
 func NewLNDMockHodlWrapperAsync(lnd lnd.LightningClientWrapper) (result *LNDMockHodlWrapperAsync, err error) {
  return &LNDMockHodlWrapperAsync{
  hps: &HodlPaymentSubscriber{
- ch: make(chan lnrpc.Payment),
+ ch: make(chan lnrpc.Payment, 5),
  },
  LightningClientWrapper: lnd,
  }, nil

integration_tests/outgoing_payment_test.go

@@ -63,7 +63,7 @@ func (suite *PaymentTestSuite) TestOutGoingPayment() {
  assert.Equal(suite.T(), 1, len(outgoingInvoices))
  assert.Equal(suite.T(), 1, len(incomingInvoices))
 
- assert.Equal(suite.T(), 3, len(transactonEntries))
+ assert.Equal(suite.T(), 5, len(transactonEntries))
 
  assert.Equal(suite.T(), int64(aliceFundingSats), transactonEntries[0].Amount)
  assert.Equal(suite.T(), currentAccount.ID, transactonEntries[0].CreditAccountID)
@@ -77,13 +77,13 @@ func (suite *PaymentTestSuite) TestOutGoingPayment() {
  assert.Equal(suite.T(), int64(0), transactonEntries[1].ParentID)
  assert.Equal(suite.T(), outgoingInvoices[0].ID, transactonEntries[1].InvoiceID)
 
- assert.Equal(suite.T(), int64(suite.mlnd.fee), transactonEntries[2].Amount)
+ assert.Equal(suite.T(), int64(suite.mlnd.fee), transactonEntries[4].Amount)
  assert.Equal(suite.T(), feeAccount.ID, transactonEntries[2].CreditAccountID)
  assert.Equal(suite.T(), currentAccount.ID, transactonEntries[2].DebitAccountID)
  assert.Equal(suite.T(), outgoingInvoices[0].ID, transactonEntries[2].InvoiceID)
 
  // make sure fee entry parent id is previous entry
- assert.Equal(suite.T(), transactonEntries[1].ID, transactonEntries[2].ParentID)
+ assert.Equal(suite.T(), transactonEntries[1].ID, transactonEntries[4].ParentID)
 
  //fetch transactions, make sure the fee is there
  // check invoices again
@@ -134,7 +134,7 @@ func (suite *PaymentTestSuite) TestOutGoingPaymentWithNegativeBalance() {
  assert.Equal(suite.T(), int64(-1), aliceBalance)
 
  // check that no additional transaction entry was created
- transactonEntries, err := suite.service.TransactionEntriesFor(context.Background(), userId)
+ transactionEntries, err := suite.service.TransactionEntriesFor(context.Background(), userId)
  if err != nil {
  fmt.Printf("Error when getting transaction entries %v\n", err.Error())
  }
@@ -149,27 +149,27 @@ func (suite *PaymentTestSuite) TestOutGoingPaymentWithNegativeBalance() {
  assert.Equal(suite.T(), 1, len(outgoingInvoices))
  assert.Equal(suite.T(), 1, len(incomingInvoices))
 
- assert.Equal(suite.T(), 3, len(transactonEntries))
+ assert.Equal(suite.T(), 5, len(transactionEntries))
 
- assert.Equal(suite.T(), int64(aliceFundingSats), transactonEntries[0].Amount)
- assert.Equal(suite.T(), currentAccount.ID, transactonEntries[0].CreditAccountID)
- assert.Equal(suite.T(), incomingAccount.ID, transactonEntries[0].DebitAccountID)
- assert.Equal(suite.T(), int64(0), transactonEntries[0].ParentID)
- assert.Equal(suite.T(), incomingInvoices[0].ID, transactonEntries[0].InvoiceID)
+ assert.Equal(suite.T(), int64(aliceFundingSats), transactionEntries[0].Amount)
+ assert.Equal(suite.T(), currentAccount.ID, transactionEntries[0].CreditAccountID)
+ assert.Equal(suite.T(), incomingAccount.ID, transactionEntries[0].DebitAccountID)
+ assert.Equal(suite.T(), int64(0), transactionEntries[0].ParentID)
+ assert.Equal(suite.T(), incomingInvoices[0].ID, transactionEntries[0].InvoiceID)
 
- assert.Equal(suite.T(), int64(externalSatRequested), transactonEntries[1].Amount)
- assert.Equal(suite.T(), outgoingAccount.ID, transactonEntries[1].CreditAccountID)
- assert.Equal(suite.T(), currentAccount.ID, transactonEntries[1].DebitAccountID)
- assert.Equal(suite.T(), int64(0), transactonEntries[1].ParentID)
- assert.Equal(suite.T(), outgoingInvoices[0].ID, transactonEntries[1].InvoiceID)
+ assert.Equal(suite.T(), int64(externalSatRequested), transactionEntries[1].Amount)
+ assert.Equal(suite.T(), outgoingAccount.ID, transactionEntries[1].CreditAccountID)
+ assert.Equal(suite.T(), currentAccount.ID, transactionEntries[1].DebitAccountID)
+ assert.Equal(suite.T(), int64(0), transactionEntries[1].ParentID)
+ assert.Equal(suite.T(), outgoingInvoices[0].ID, transactionEntries[1].InvoiceID)
 
- assert.Equal(suite.T(), int64(suite.mlnd.fee), transactonEntries[2].Amount)
- assert.Equal(suite.T(), feeAccount.ID, transactonEntries[2].CreditAccountID)
- assert.Equal(suite.T(), currentAccount.ID, transactonEntries[2].DebitAccountID)
- assert.Equal(suite.T(), outgoingInvoices[0].ID, transactonEntries[2].InvoiceID)
+ assert.Equal(suite.T(), int64(suite.mlnd.fee), transactionEntries[4].Amount)
+ assert.Equal(suite.T(), feeAccount.ID, transactionEntries[2].CreditAccountID)
+ assert.Equal(suite.T(), currentAccount.ID, transactionEntries[2].DebitAccountID)
+ assert.Equal(suite.T(), outgoingInvoices[0].ID, transactionEntries[2].InvoiceID)
 
  // make sure fee entry parent id is previous entry
- assert.Equal(suite.T(), transactonEntries[1].ID, transactonEntries[2].ParentID)
+ assert.Equal(suite.T(), transactionEntries[1].ID, transactionEntries[4].ParentID)
 }
 
 func (suite *PaymentTestSuite) TestZeroAmountInvoice() {