github/weave-scope
1
0
Fork 0
mirror of https://github.com/weaveworks/scope.git synced 2026-03-04 10:41:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make scheduler aware of test parallelisation

Browse Source
This commit is contained in:
Tom Wilkie
2016-05-18 14:08:05 +01:00
parent 7a660904a6
commit e9749a5383
3 changed files with 30 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -2,3 +2,5 @@ cover/cover
socks/proxy socks/proxy
socks/image.tar socks/image.tar
runner/runner runner/runner
*.pyc
*~

26
sched
View File

@@ -1,36 +1,38 @@
#!/usr/bin/python #!/usr/bin/python
import sys, string, json, urllib import sys, string, json, urllib
import requests import requests
import optparse
BASE_URL="http://positive-cocoa-90213.appspot.com" def test_time(target, test_name, runtime):
r = requests.post(target + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime))
def test_time(test_name, runtime):
r = requests.post(BASE_URL + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime))
print r.text print r.text
assert r.status_code == 204 assert r.status_code == 204
def test_sched(test_run, shard_count, shard_id): def test_sched(target, test_run, shard_count, shard_id):
tests = json.dumps({'tests': string.split(sys.stdin.read())}) tests = json.dumps({'tests': string.split(sys.stdin.read())})
r = requests.post(BASE_URL + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), data=tests) r = requests.post(target + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), data=tests)
assert r.status_code == 200 assert r.status_code == 200
result = r.json() result = r.json()
for test in sorted(result['tests']): for test in sorted(result['tests']):
print test print test
def usage(): def usage():
print "%s <cmd> <args..>" % sys.argv[0] print "%s (--target=...) <cmd> <args..>" % sys.argv[0]
print " time <test name> <run time>" print " time <test name> <run time>"
print " sched <test run> <num shards> <shard id>" print " sched <test run> <num shards> <shard id>"
def main(): def main():
if len(sys.argv) < 4: parser = optparse.OptionParser()
parser.add_option('--target', default="http://positive-cocoa-90213.appspot.com")
options, args = parser.parse_args()
if len(args) < 3:
usage() usage()
sys.exit(1) sys.exit(1)
if sys.argv[1] == "time": if args[0] == "time":
test_time(sys.argv[2], float(sys.argv[3])) test_time(options.target, args[1], float(args[2]))
elif sys.argv[1] == "sched": elif args[0] == "sched":
test_sched(sys.argv[2], int(sys.argv[3]), int(sys.argv[4])) test_sched(options.target, args[1], int(args[2]), int(args[3]))
else: else:
usage() usage()

15
scheduler/main.py
View File

@@ -23,6 +23,19 @@ class Test(ndb.Model):
total_run_time = ndb.FloatProperty(default=0.) # Not total, but a EWMA total_run_time = ndb.FloatProperty(default=0.) # Not total, but a EWMA
total_runs = ndb.IntegerProperty(default=0) total_runs = ndb.IntegerProperty(default=0)
def parallelism(self):
name = self.key.string_id()
m = re.search('(\d+)_test.sh$', name)
if m is None:
return 1
else:
return int(m.group(1))
def cost(self):
p = self.parallelism()
logging.info("Test %s has parallelism %d and avg run time %s", self.key.string_id(), p, self.total_run_time)
return self.parallelism() * self.total_run_time
class Schedule(ndb.Model): class Schedule(ndb.Model):
shards = ndb.JsonProperty() shards = ndb.JsonProperty()
@@ -52,7 +65,7 @@ def schedule(test_run, shard_count, shard):
test_times = ndb.get_multi(ndb.Key(Test, test_name) for test_name in test_names) test_times = ndb.get_multi(ndb.Key(Test, test_name) for test_name in test_names)
def avg(test): def avg(test):
if test is not None: if test is not None:
return test.total_run_time return test.cost()
return 1 return 1
test_times = [(test_name, avg(test)) for test_name, test in zip(test_names, test_times)] test_times = [(test_name, avg(test)) for test_name, test in zip(test_names, test_times)]
test_times_dict = dict(test_times) test_times_dict = dict(test_times)