import json import os from getpass import getpass from pathlib import Path from typing import Dict, List, Optional, Union from IPython.display import display, HTML, JSON try: import google.colab IN_COLAB = True except ModuleNotFoundError: IN_COLAB = False DRIVE = ( Path("/content/drive/MyDrive") if IN_COLAB and Path("/content/drive/MyDrive").exists() else None ) COLAB = None if DRIVE: COLAB = DRIVE / "colab" if DRIVE is not None else None os.environ["GDRIVE"] = str(DRIVE) os.environ["COLAB"] = str(COLAB) def rdict(d: Dict) -> None: display(JSON(d)) def rhtml(html: str) -> None: display(HTML(html)) def rimg(img_url: str, width: int = None, height: int = None) -> None: display(HTML(f"")) def rheading(text: str, level=1) -> None: rhtml(f"{text}") def load_secrets( secrets: Optional[Union[str, Dict[str, str]]] = None, overwrite=False ) -> None: """ Loads secrets and sets up some env vars and credential files. If the `secrets` param is empty you will be prompted to input a stringified json dict containing your secrets. The following types of credentials are supported: GitHub Credentials: `github_user`: GitHub Username `github_pat`: GitHub Personal Access Token AWS Credentials: `aws_access_key_id`: AWS Key ID `aws_secret_access_key`: AWS Access Key """ if secrets and isinstance(secrets, str): secrets = json.loads(secrets) if not secrets: input = getpass("Secrets (JSON string): ") secrets = json.loads(input) if input else {} if "github_pat" in secrets: os.environ["GH_USER"] = secrets["github_user"] os.environ["GH_PAT"] = secrets["github_pat"] os.environ["GITHUB_TOKEN"] = secrets["github_pat"] if "aws_access_key_id" in secrets: home = Path.home() aws_id = secrets["aws_access_key_id"] aws_key = secrets["aws_secret_access_key"] (home / ".aws/").mkdir(parents=True, exist_ok=True) with open(home / ".aws/credentials", "w") as fp: fp.write( f"[default]\naws_access_key_id = {aws_id}\naws_secret_access_key = {aws_key}\n" ) if "hf_token" in secrets: with open("/root/.huggingface/token", "w") as fp: fp.write(secrets["hf_token"]) if "wandb_token" in secrets: netrc = ( f"machine api.wandb.ai\n\tlogin user\n\tpassword {secrets['wandb_token']}" ) with open("/root/.netrc", "w") as fp: fp.write(netrc) def wget(urls: Union[str, List[str]], silent=True): """ Use wget to download a list of urls. """ if isinstance(urls, str): urls = [urls] for url in urls: if "dropbox.com" in url: url = url.split("?")[0] out_name = url.split("?")[0].rstrip("/").split("/")[-1] silent_flag = "-q" if silent else "" os.system(f"wget {silent_flag} {url} -O {out_name}") def git_clone(repos: Union[str, List[str]]): user = os.environ.get("GH_USER") pat = os.environ.get("GH_PAT") if user: assert ( pat ), "please call load_secrets() first and provide both `github_user` and `github_pat`" if isinstance(repos, str): repos = [repos] for repo_name in repos: if ".git" not in repo_name: if "@" in repo_name: repo_name = repo_name.replace("@", ".git@") else: repo_name += ".git" if user: os.system(f"git clone https://{user}:{pat}@github.com/{repo_name}") else: os.system(f"git clone https://github.com/{repo_name}") def pip_install(*, packages: Union[str, List[str]], silent=True, force_install=False): user = os.environ.get("GH_USER") if isinstance(packages, str): packages = [packages] def _get_package_name(full_descriptor): return ( full_descriptor if "/" not in full_descriptor else full_descriptor.split("/")[-1].split("@")[0] ) package_names = " ".join( [ (pkg.split("/")[-1].split("@")[0] + "/") if "/" in pkg else pkg for pkg in packages ] ) # install github repos git_clone(user=user, repos=[pkg for pkg in packages if "/" in pkg]) # install packages silent_flag = "-qqq" if silent else "" os.system(f"pip install {silent_flag} {package_names}") def report_gpu(): # import GPUtil # for i, gpu in enumerate(GPUtil.getGPUs()): # print(f'[GPU {i}] Mem Free: {gpu.memoryFree/1024:.2f}/{gpu.memoryTotal/1024:.2f} GB | Utilization: {int(gpu.memoryUtil*100)}%') import torch if torch.cuda.is_available(): name = torch.cuda.get_device_name(0) print( f"[{name}] RAM: {torch.cuda.memory_allocated(0)/1024**3:.2f}/{torch.cuda.memory_reserved(0)/1024**3:.2f} GB used" ) if IN_COLAB: load_secrets() report_gpu()