here is my python script, to process about 1m files, mainly to check if these files exists:

def check_data_valid():
    data_root = "/data_alpha/data/video/internvid/"
    save_root = "/data_alpha/data/video/internvid/all_file_list/proc"
    download_list = "/data_alpha/data/video/internvid/all_file_list/all_files.txt"
    valid_data = []
    err_data = []
    missing_data = []
    with open(download_list, "r") as f:
        a = f.readlines()
        downloads = [i.strip() for i in a]
        for i, item in enumerate(downloads):
            file = os.path.join(data_root, item)
            avi = file.replace(".mp4", ".avi")
            if os.path.exists(avi):
                valid_data.append(item)
            elif os.path.exists(file):
                err_data.append(item)
            else:
                missing_data.append(item)
            if (i) % 200000 == 1:
                if len(valid_data) > 0:
                    with open(os.path.join(save_root, f"valid_{len(valid_data)}_{i}.txt"), "w") as f:
                        f.writelines("\n".join(valid_data))
                if len(err_data) > 0:
                    with open(os.path.join(save_root, f"error_{len(err_data)}_{i+start}.txt"), "w") as f:
                        f.writelines("\n".join(err_data))
                if len(missing_data) > 0:
                    with open(os.path.join(save_root, f"missing_{len(missing_data)}_{i}.txt"), "w") as f:
                        f.writelines("\n".join(missing_data))
        if len(valid_data) > 0:
            with open(os.path.join(save_root, f"valid_total.txt"), "w") as f:
                f.writelines("\n".join(valid_data))
        if len(err_data) > 0:
            with open(os.path.join(save_root, f"error_total.txt"), "w") as f:
                f.writelines("\n".join(err_data))
        if len(missing_data) > 0:
            with open(os.path.join(save_root, f"missing_total.txt"), "w") as f:
                f.writelines("\n".join(missing_data))



if __name__ == '__main__':
    check_data_valid()

but after run serveral hours, and processed about 100k files by reviewing the files in save_root. Then this process will turn into status D (by use ps command to see), which means Uninterruptible Sleep, I wonder why this happens and how to avoid this.