TRA-3437 prevent vacuum bottleneck

TRA-3437 prevent vacuum bottleneck

api/pkg/api/main.go

@@ -161,7 +161,7 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
 		IsOutgoing:          connectionInfo.IsOutgoing,
 	}
 	mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry)
-	database.GetEntriesTable().Create(&mizuEntry)
+	database.CreateEntry(&mizuEntry)
 
 	baseEntry := models.BaseEntryDetails{}
 	if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil {

api/pkg/database/main.go

@@ -12,11 +12,6 @@ import (
 
 const (
 	DBPath = "./entries.db"
-)
-
- var DB *gorm.DB
-
-const (
 	OrderDesc = "desc"
 	OrderAsc  = "asc"
 	LT        = "lt"
@@ -24,6 +19,8 @@ const (
 )
 
 var (
+	DB *gorm.DB
+	IsDBLocked = false
 	OperatorToSymbolMapping = map[string]string{
 		LT: "<",
 		GT: ">",
@@ -43,6 +40,13 @@ func GetEntriesTable() *gorm.DB {
 	return DB.Table("mizu_entries")
 }
 
+func CreateEntry(entry *models.MizuEntry) {
+	if IsDBLocked {
+		return
+	}
+	GetEntriesTable().Create(entry)
+}
+
 func initDataBase(databasePath string) *gorm.DB {
 	temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{
 		Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond},

api/pkg/database/size_enforcer.go

@@ -81,6 +81,10 @@ func checkFileSize(maxSizeBytes int64) {
 }
 
 func pruneOldEntries(currentFileSize int64) {
+	// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
+	IsDBLocked = true
+	defer func() {IsDBLocked = false}()
+
 	amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune)
 
 	rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()

cli/cmd/tap.go

@@ -64,10 +64,8 @@ Supported protocols are HTTP and gRPC.`,
 		mizuTapOptions.MaxEntriesDBSizeBytes, parseHumanDataSizeErr = units.HumanReadableToBytes(humanMaxEntriesDBSize)
 		if parseHumanDataSizeErr != nil {
 			return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize))
-		} else if cmd.Flags().Changed(maxEntriesDBSizeFlagName) {
-			// We're parsing human readable file sizes here so its best to be unambiguous
-			fmt.Printf("Setting max entries db size to %s\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes))
 		}
+		fmt.Printf("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes))
 
 		directionLowerCase := strings.ToLower(direction)
 		if directionLowerCase == "any" {