Spaces:
Sleeping
Sleeping
| from google_sheet import conf_repo, secret_repo | |
| import json | |
| from kaggle_service import KernelRerunService, NbJob | |
| from logger import sheet_logger | |
| if __name__ == "__main__": | |
| sheet_logger.log("========= start ==========") | |
| # load all account and secret into a dict | |
| secret_dict = {} | |
| try: | |
| for i in range(2, 50): # note: read row 2 - 50 | |
| rs = secret_repo.read(i) | |
| if not rs: | |
| break | |
| assert 'username' in rs | |
| assert 'secret' in rs | |
| username = rs['username'].rstrip() | |
| secret = rs['secret'].rstrip() | |
| secret_dict[username] = secret | |
| except Exception as e: | |
| sheet_logger.log(log="Get secret table failed!!" + str(e)) | |
| # load config from gg sheet file | |
| configs = [] | |
| ids = [] | |
| try: | |
| for i in range(2, 50): # note: read row 2 - 50 | |
| rs = conf_repo.read(i) | |
| if not rs: | |
| break | |
| if not rs['config']: | |
| reset_keys = [] | |
| for key in rs: | |
| if rs[key]: | |
| reset_keys.append(key) | |
| if len(reset_keys) > 0: | |
| conf_repo.update(row_index=i, data={k: '' for k in reset_keys}) | |
| break | |
| print(i, rs) | |
| try: | |
| # validation | |
| # assert 'config' in rs, 'require config column' | |
| assert 'usernames' in rs, 'require usernames column' | |
| assert 'enable' in rs, 'require enable column' | |
| assert rs['enable'].rstrip() in ['enable', 'disable'], rs['enable'] | |
| assert 'slug' in rs['config'], 'require config.slug' | |
| cfg = json.loads(rs['config']) | |
| usernames = set(rs['usernames'] | |
| .rstrip().replace('\n', ' ').replace(',', ' ').replace(';', ' ').replace('|', ' ') | |
| .split(' ')) | |
| usernames = [u for u in usernames if u] | |
| is_enabled = rs['enable'].rstrip() == 'enable' | |
| if not is_enabled: | |
| sheet_logger.log(log="Disabled, skip!", nb=cfg['slug']) | |
| print("skip ", cfg['slug']) | |
| continue | |
| assert len(usernames) > 0, 'len usernames == 0' | |
| # process config | |
| accounts = {u: secret_dict[u] for u in secret_dict if u in usernames} | |
| assert not set(usernames).difference(set(accounts.keys())), set(usernames).difference(set(accounts.keys())) | |
| cfg = {**cfg, "accounts": accounts} | |
| # save | |
| configs.append(cfg) | |
| ids.append(i) | |
| print(cfg) | |
| except AssertionError: | |
| import traceback | |
| sheet_logger.update_job_status(i, validate_status=str(traceback.format_exc())) | |
| except Exception: | |
| import traceback | |
| sheet_logger.log(log="Get config failed!!" + str(traceback.format_exc())) | |
| # add config to service | |
| service = KernelRerunService() | |
| for config, idx in zip(configs, ids): | |
| try: | |
| service.add_job(NbJob.from_dict(config, id=idx)) | |
| except Exception as e: | |
| sheet_logger.update_job_status(idx, validate_status=str(e)) | |
| # run service | |
| try: | |
| service.validate_all() | |
| # service.status_all() | |
| service.run_all() | |
| except Exception as e: | |
| sheet_logger.log(log=str(e)) | |
| sheet_logger.log("========= end ==========") | |