Just gonna open an issue on myself as I assume people will be most likely to see it here. =)
If you get this error, it has nothing to do with my code / node. It's an issue with PyTorch. To fix it:
def write_atomic(
path_: str,
content: Union[str, bytes],
make_dirs: bool = False,
encode_utf_8: bool = False,
) -> None:
# Write into temporary file first to avoid conflicts between threads
# Avoid using a named temporary file, as those have restricted permissions
assert isinstance(
content, (str, bytes)
), "Only strings and byte arrays can be saved in the cache"
path = Path(path_)
if make_dirs:
path.parent.mkdir(parents=True, exist_ok=True)
tmp_path = path.parent / f".{os.getpid()}.{threading.get_ident()}.tmp"
write_mode = "w" if isinstance(content, str) else "wb"
with tmp_path.open(write_mode, encoding="utf-8" if encode_utf_8 else None) as f:
f.write(content)
#tmp_path.rename(path) # comment this out, and add the following two lines instead:
shutil.copy2(src=tmp_path, dst=path)
os.remove(tmp_path)