Spaces:
Sleeping
Sleeping
File size: 3,539 Bytes
2b0538d 2a8b18d edf61c6 2b0538d edf61c6 2b0538d edf61c6 2b0538d edf61c6 2b0538d edf61c6 2b0538d edf61c6 2b0538d 2a8b18d 2b0538d edf61c6 2a8b18d 2b0538d edf61c6 2a8b18d 2b0538d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 |
from google_sheet import conf_repo, secret_repo
import json
from kaggle_service import KernelRerunService, NbJob
from logger import sheet_logger
if __name__ == "__main__":
sheet_logger.log("========= start ==========")
# load all account and secret into a dict
secret_dict = {}
try:
for i in range(2, 50): # note: read row 2 - 50
rs = secret_repo.read(i)
if not rs:
break
assert 'username' in rs
assert 'secret' in rs
username = rs['username'].rstrip()
secret = rs['secret'].rstrip()
secret_dict[username] = secret
except Exception as e:
sheet_logger.log(log="Get secret table failed!!" + str(e))
# load config from gg sheet file
configs = []
ids = []
try:
for i in range(2, 50): # note: read row 2 - 50
rs = conf_repo.read(i)
if not rs:
break
if not rs['config']:
reset_keys = []
for key in rs:
if rs[key]:
reset_keys.append(key)
if len(reset_keys) > 0:
conf_repo.update(row_index=i, data={k: '' for k in reset_keys})
break
print(i, rs)
try:
# validation
# assert 'config' in rs, 'require config column'
assert 'usernames' in rs, 'require usernames column'
assert 'enable' in rs, 'require enable column'
assert rs['enable'].rstrip() in ['enable', 'disable'], rs['enable']
assert 'slug' in rs['config'], 'require config.slug'
cfg = json.loads(rs['config'])
usernames = set(rs['usernames']
.rstrip().replace('\n', ' ').replace(',', ' ').replace(';', ' ').replace('|', ' ')
.split(' '))
usernames = [u for u in usernames if u]
is_enabled = rs['enable'].rstrip() == 'enable'
if not is_enabled:
sheet_logger.log(log="Disabled, skip!", nb=cfg['slug'])
print("skip ", cfg['slug'])
continue
assert len(usernames) > 0, 'len usernames == 0'
# process config
accounts = {u: secret_dict[u] for u in secret_dict if u in usernames}
assert not set(usernames).difference(set(accounts.keys())), set(usernames).difference(set(accounts.keys()))
cfg = {**cfg, "accounts": accounts}
# save
configs.append(cfg)
ids.append(i)
print(cfg)
except AssertionError:
import traceback
sheet_logger.update_job_status(i, validate_status=str(traceback.format_exc()))
except Exception:
import traceback
sheet_logger.log(log="Get config failed!!" + str(traceback.format_exc()))
# add config to service
service = KernelRerunService()
for config, idx in zip(configs, ids):
try:
service.add_job(NbJob.from_dict(config, id=idx))
except Exception as e:
sheet_logger.update_job_status(idx, validate_status=str(e))
# run service
try:
service.validate_all()
# service.status_all()
service.run_all()
except Exception as e:
sheet_logger.log(log=str(e))
sheet_logger.log("========= end ==========")
|