Refactor: extract deleteOneOrgHour() method

This commit is contained in:
Bryan Boreham
2019-06-04 15:13:48 +00:00
parent 388d273e5f
commit cdf28708e8
+33 -29
View File
@@ -211,39 +211,43 @@ func main() {
func (sc *scanner) processOrg(ctx context.Context, org string) {
deleted := 0
for hour := sc.startHour; hour <= sc.stopHour; hour++ {
var keys []map[string]*dynamodb.AttributeValue
for {
sc.queryLimiter.Wait(ctx)
var err error
keys, err = queryDynamo(ctx, sc.dynamoDB, sc.tableName, org, int64(hour))
if throttled(err) {
continue
}
checkFatal(err)
break
}
var wait sync.WaitGroup
if len(keys) > 0 {
log.Debugf("deleting org: %s hour: %d num: %d", org, hour, len(keys))
}
for start := 0; start < len(keys); start += s3deleteBatchSize {
end := start + s3deleteBatchSize
if end > len(keys) {
end = len(keys)
}
wait.Add(1)
go func(start, end int) {
sc.deleteFromS3AndDynamoDB(ctx, keys[start:end])
wait.Done()
}(start, end)
}
wait.Wait()
deleted += len(keys)
s3ItemsDeleted.Add(float64(len(keys)))
deleted += sc.deleteOneOrgHour(ctx, org, hour)
}
log.Infof("done %s: %d", org, deleted)
}
func (sc *scanner) deleteOneOrgHour(ctx context.Context, org string, hour int) int {
var keys []map[string]*dynamodb.AttributeValue
for {
sc.queryLimiter.Wait(ctx)
var err error
keys, err = queryDynamo(ctx, sc.dynamoDB, sc.tableName, org, int64(hour))
if throttled(err) {
continue
}
checkFatal(err)
break
}
var wait sync.WaitGroup
if len(keys) > 0 {
log.Debugf("deleting org: %s hour: %d num: %d", org, hour, len(keys))
}
for start := 0; start < len(keys); start += s3deleteBatchSize {
end := start + s3deleteBatchSize
if end > len(keys) {
end = len(keys)
}
wait.Add(1)
go func(start, end int) {
sc.deleteFromS3AndDynamoDB(ctx, keys[start:end])
wait.Done()
}(start, end)
}
wait.Wait()
s3ItemsDeleted.Add(float64(len(keys)))
return len(keys)
}
func (sc *scanner) deleteFromS3AndDynamoDB(ctx context.Context, keys []map[string]*dynamodb.AttributeValue) {
// Build multiple-object delete request for S3
d := &s3.Delete{}