Files
11b731c65a Initial commit
Co-authored-by: Nicolas Ballas <[email protected]>
Co-authored-by: Adrien Bardes <[email protected]>
Co-authored-by: David Fan <[email protected]>
Co-authored-by: Quentin Garrido <[email protected]>
Co-authored-by: Russell Howes <[email protected]>
Co-authored-by: Mojtaba Komeili <[email protected]>
Co-authored-by: Matthew Muckley <[email protected]>
Co-authored-by: Koustuv Sinha <[email protected]>
Co-authored-by: Artem Zholus <[email protected]>
2025-06-11 05:50:50 -07:00

85 lines
2.5 KiB
Python

# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import argparse
import multiprocessing as mp
import pprint
from pathlib import Path
import yaml
from app.scaffold import main as app_main
from src.utils.distributed import init_distributed
parser = argparse.ArgumentParser()
parser.add_argument("--fname", type=str, help="name of config file to load", default="configs.yaml")
parser.add_argument(
"--devices",
type=str,
nargs="+",
default=["cuda:0", "cuda:1", "cuda:2", "cuda:3", "cuda:4", "cuda:5", "cuda:6", "cuda:7"],
help="which devices to use on local machine",
)
parser.add_argument(
"--debugmode",
type=bool,
default=False,
help="Setting this to true will not spin up new processes. "
"The main code runs the main process, which makes it easier to \
debug with checkpointing.",
)
def process_main(rank, fname, world_size, devices):
import os
os.environ["CUDA_VISIBLE_DEVICES"] = str(devices[rank].split(":")[-1])
import logging
from src.utils.logging import get_logger
logger = get_logger(force=True)
if rank == 0:
logger.setLevel(logging.INFO)
else:
logger.setLevel(logging.ERROR)
logger.info(f"called-params {fname}")
# Load config
params = None
with open(fname, "r") as y_file:
params = yaml.load(y_file, Loader=yaml.FullLoader)
logger.info("loaded params...")
# Log config
if rank == 0:
pprint.PrettyPrinter(indent=4).pprint(params)
folder = params["folder"]
params_path = os.path.join(folder, "params-pretrain.yaml")
folder = Path(folder)
folder.mkdir(parents=True, exist_ok=True)
with open(params_path, "w") as f:
yaml.dump(params, f)
# Init distributed (access to comm between GPUS on same machine)
world_size, rank = init_distributed(rank_and_world_size=(rank, world_size))
logger.info(f"Running... (rank: {rank}/{world_size})")
# Launch the app with loaded config
app_main(params["app"], args=params)
if __name__ == "__main__":
args = parser.parse_args()
if args.debugmode:
process_main(rank=0, fname=args.fname, world_size=1, devices=["cuda:0"])
else:
num_gpus = len(args.devices)
mp.set_start_method("spawn")
for rank in range(num_gpus):
mp.Process(target=process_main, args=(rank, args.fname, num_gpus, args.devices)).start()