diff --git a/app/multitenant/sqs_control_router.go b/app/multitenant/sqs_control_router.go index 35071aef3..d9a4c93b9 100644 --- a/app/multitenant/sqs_control_router.go +++ b/app/multitenant/sqs_control_router.go @@ -22,7 +22,6 @@ import ( var ( longPollTime = aws.Int64(10) - rpcTimeout = time.Minute sqsRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "scope", Name: "sqs_request_duration_seconds", @@ -45,6 +44,7 @@ type sqsControlRouter struct { responseQueueURL *string userIDer UserIDer prefix string + rpcTimeout time.Duration mtx sync.Mutex responses map[string]chan xfer.Response @@ -63,12 +63,13 @@ type sqsResponseMessage struct { } // NewSQSControlRouter the harbinger of death -func NewSQSControlRouter(config *aws.Config, userIDer UserIDer, prefix string) app.ControlRouter { +func NewSQSControlRouter(config *aws.Config, userIDer UserIDer, prefix string, rpcTimeout time.Duration) app.ControlRouter { result := &sqsControlRouter{ service: sqs.New(session.New(config)), responseQueueURL: nil, userIDer: userIDer, prefix: prefix, + rpcTimeout: rpcTimeout, responses: map[string]chan xfer.Response{}, probeWorkers: map[int64]*probeWorker{}, } @@ -257,7 +258,7 @@ func (cr *sqsControlRouter) Handle(ctx context.Context, probeID string, req xfer select { case response := <-waiter: return response, nil - case <-time.After(rpcTimeout): + case <-time.After(cr.rpcTimeout): return xfer.Response{}, fmt.Errorf("request timed out") } } diff --git a/prog/app.go b/prog/app.go index 7b517100e..e9427d827 100644 --- a/prog/app.go +++ b/prog/app.go @@ -147,7 +147,7 @@ func emitterFactory(collector app.Collector, clientCfg billing.Config, userIDer ) } -func controlRouterFactory(userIDer multitenant.UserIDer, controlRouterURL string) (app.ControlRouter, error) { +func controlRouterFactory(userIDer multitenant.UserIDer, controlRouterURL string, controlRPCTimeout time.Duration) (app.ControlRouter, error) { if controlRouterURL == "local" { return app.NewLocalControlRouter(), nil } @@ -163,7 +163,7 @@ func controlRouterFactory(userIDer multitenant.UserIDer, controlRouterURL string if err != nil { return nil, err } - return multitenant.NewSQSControlRouter(sqsConfig, userIDer, prefix), nil + return multitenant.NewSQSControlRouter(sqsConfig, userIDer, prefix, controlRPCTimeout), nil } return nil, fmt.Errorf("Invalid control router '%s'", controlRouterURL) @@ -239,7 +239,7 @@ func appMain(flags appFlags) { collector = billingEmitter } - controlRouter, err := controlRouterFactory(userIDer, flags.controlRouterURL) + controlRouter, err := controlRouterFactory(userIDer, flags.controlRouterURL, flags.controlRPCTimeout) if err != nil { log.Fatalf("Error creating control router: %v", err) return diff --git a/prog/main.go b/prog/main.go index f8099bc60..390171f1e 100644 --- a/prog/main.go +++ b/prog/main.go @@ -152,6 +152,7 @@ type appFlags struct { collectorURL string s3URL string controlRouterURL string + controlRPCTimeout time.Duration pipeRouterURL string natsHostname string memcachedHostname string @@ -348,6 +349,7 @@ func setupFlags(flags *flags) { flag.StringVar(&flags.app.collectorURL, "app.collector", "local", "Collector to use (local, dynamodb, or file/directory)") flag.StringVar(&flags.app.s3URL, "app.collector.s3", "local", "S3 URL to use (when collector is dynamodb)") flag.StringVar(&flags.app.controlRouterURL, "app.control.router", "local", "Control router to use (local or sqs)") + flag.DurationVar(&flags.app.controlRPCTimeout, "app.control.rpctimeout", time.Minute, "Timeout for control RPC") flag.StringVar(&flags.app.pipeRouterURL, "app.pipe.router", "local", "Pipe router to use (local)") flag.StringVar(&flags.app.natsHostname, "app.nats", "", "Hostname for NATS service to use for shortcut reports. If empty, shortcut reporting will be disabled.") flag.StringVar(&flags.app.memcachedHostname, "app.memcached.hostname", "", "Hostname for memcached service to use when caching reports. If empty, no memcached will be used.")